Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<feature_engineering>
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<data_type_conversions>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
for col in('GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )<categorify>
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 3
MAX_LEN = 220
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<count_missing_values>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv',low_memory=True)
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv',low_memory=True )
null_columns=all_data.columns[all_data.isnull().any() ]
all_data[null_columns].isnull().sum()<categorify>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
cols =('BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'CentralAir')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))<feature_engineering>
x_train = preprocess(train['comment_text'])
 y_train = np.where(train['target'] >= 0.5, 1, 0 )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<sort_values>
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'] )
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10 )<feature_engineering>
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )<categorify>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
train = all_data[:ntrain]
test = all_data[ntrain:]<import_modules>
del train
gc.collect()
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<split>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
y = df_train['SalePrice']
y_train = y
X_train_sparse, X_test_sparse, y_train_sparse, y_test_sparse = train_test_split(
 train, y_train,
test_size=0.25,
random_state=42
)<compute_train_metric>
def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 score = rmsle_cv(model_xgb)
 regression(model_xgb,X_test_sparse,y_test_sparse)
 regressionPlot(model_xgb,X_test_sparse,y_test_sparse,"XGBoost" )<choose_model_class>
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model_lgb)
 regression(model_lgb,X_test_sparse,y_test_sparse)
 regressionPlot(model_lgb,X_test_sparse,y_test_sparse,"LightGBM" )<train_model>
del tokenizer
gc.collect()
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
checkpoint_predictions = []
weights = []
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<compute_train_metric>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )
<predict_on_test><EOS> 
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )