Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<feature_engineering>
start_time = time.time()
np.random.seed(32)
 os.environ["OMP_NUM_THREADS"] = "4"
df['X118_plus_X314_plus_X315'] = df.apply(lambda row: row.X118 + row.X314 + row.X315, axis=1)
 test['X118_plus_X314_plus_X315'] = test.apply(lambda row: row.X118 + row.X314 + row.X315, axis=1 )<feature_engineering>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
df["X10_plus_X54"] = df.apply(lambda row: row.X10 + row.X54, axis=1)
 test["X10_plus_X54"] = test.apply(lambda row: row.X10 + row.X54, axis=1)
 print("Correalation between X10_plus_X54 and y is : ",df.y.corr(df['X10_plus_X54']))<feature_engineering>
train = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv")
 embedding_path = ".. /input/glove840b300dtxt/glove.840B.300d.txt"
embed_size = 300
max_features = 100000
max_len = 150
df["X10_plus_X29"] = df.apply(lambda row: row.X10 + row.X29, axis=1)
 test["X10_plus_X29"] = test.apply(lambda row: row.X10 + row.X29, axis=1)
 print("Correalation between X10_plus_X29 and y is : ",df.y.corr(df['X10_plus_X29']))<feature_engineering>
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
train["comment_text"].fillna("no comment")
 test["comment_text"].fillna("no comment")
 X_train, X_valid, Y_train, Y_valid = train_test_split(train, y, test_size = 0.1 )
train_X['X314_plus_X315']=df['X314_plus_X315']
train_X['X118_plus_X314_plus_X315']=df['X118_plus_X314_plus_X315']
train_X["X10_plus_X54"] = df["X10_plus_X54"]
train_X["X10_plus_X29"] = df["X10_plus_X29"]<drop_column>
raw_text_train = X_train["comment_text"].str.lower()
raw_text_valid = X_valid["comment_text"].str.lower()
raw_text_test = test["comment_text"].str.lower()
tk = Tokenizer(num_words = max_features, lower = True)
 tk.fit_on_texts(raw_text_train)
 X_train["comment_seq"] = tk.texts_to_sequences(raw_text_train)
 X_valid["comment_seq"] = tk.texts_to_sequences(raw_text_valid)
 test["comment_seq"] = tk.texts_to_sequences(raw_text_test)
 X_train = pad_sequences(X_train.comment_seq, maxlen = max_len)
 X_valid = pad_sequences(X_valid.comment_seq, maxlen = max_len)
 test = pad_sequences(test.comment_seq, maxlen = max_len )
booler = np.ones(400)
 for i in same_features:
if booler[int(i[1][1:])]==1:
booler[int(i[1][1:])]=0
df_num.drop(i[1],axis=1,inplace=True)
 df.drop(i[1],axis=1,inplace=True)
 test.drop(i[1],axis=1,inplace=True)
 train_X.drop(i[1],axis=1,inplace=True)
 elif booler[int(i[0][1:])]==1:
booler[int(i[0][1:])]=0
df_num.drop(i[0],axis=1,inplace=True)
 df.drop(i[0],axis=1,inplace=True)
 test.drop(i[0],axis=1,inplace=True)
 train_X.drop(i[0],axis=1,inplace=True )<categorify>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))
<categorify>
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
<split>
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 ra_val = RocAucEvaluation(validation_data=(X_valid, Y_valid), interval = 1)
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 5)
 def build_model(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
inp = Input(shape =(max_len,))
x = Embedding(max_features, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x = SpatialDropout1D(dr )(x)
 x = Bidirectional(GRU(units, return_sequences = True))(x)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 x = Dense(6, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 history = model.fit(X_train, Y_train, batch_size = 128, epochs = 4, validation_data =(X_valid, Y_valid),
verbose = 1, callbacks = [ra_val, check_point, early_stop])
 model = load_model(file_path)
 return model
x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=420)
 d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(test)
 xgb_params = {
'n_trees': 500,
'eta': 0.0050,
'max_depth': 3,
'subsample': 0.95,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
def xgb_r2_score(preds, dtrain):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(xgb_params, d_train, 1050 , watchlist, early_stopping_rounds=70, feval=xgb_r2_score, maximize=True, verbose_eval=10 )<train_model>
model = build_model(lr = 1e-3, lr_d = 0, units = 128, dr = 0.2)
 pred = model.predict(test, batch_size = 1024, verbose = 1 )
d_train = xgb.DMatrix(train_X, label=train_y)
 d_test = xgb.DMatrix(test)
 xgb_params = {
'n_trees': 500,
'eta': 0.0050,
'max_depth': 3,
'subsample': 0.95,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
def xgb_r2_score(preds, dtrain):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 watchlist = [(d_train, 'train')]
clf = xgb.train(xgb_params, d_train, 1050 , watchlist, early_stopping_rounds=70, feval=xgb_r2_score, maximize=True, verbose_eval=10 )<save_to_csv>
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False)
 print("[{}] Completed!".format(time.time() - start_time))