Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<train_on_grid>
print(os.listdir(".. /input"))
print(os.listdir(".. /input/jigsaw-toxic-comment-classification-challenge"))
if run_mode == 'grid_search_RFR':
perc_samples = 0.15
print("
Preparing to run Hyperparameters tunning with GridSearchCV using {0:.2f}% of the training samples".format(perc_samples * 100))
features_train_small = X_train[:int(perc_samples * X_train.shape[0])]
target_train_small = y_train[:int(perc_samples * y_train.shape[0])]
features_val_small = X_val[:int(perc_samples * X_val.shape[0])]
target_val_small = y_val[:int(perc_samples * y_val.shape[0])]
estimator = RandomForestRegressor(n_jobs=-1,
random_state=42,
verbose=0)

parameters = {
'n_estimators': [130, 135, 145],
'min_samples_leaf': [55, 62, 75],
'max_features': [0.2],
'min_samples_split': [2],
}
scorer = make_scorer(roc_auc_score)
 grid_obj = GridSearchCV(estimator=estimator,
param_grid=parameters,
scoring=scorer)
 start = time()
grid_fit = grid_obj.fit(features_train_small, target_train_small)
 end = time()
grid_fit_time =(end - start)/ 60
print("
GridSearchCV estimator fit time: {0:.2f} minutes".format(( end - start)/ 60))
best_est = grid_obj.best_estimator_
print("
Best Estimator:
{}
".format(best_est))
best_score = grid_obj.best_score_
print("
Best Estimator Score: {}
".format(best_score))
best_params = grid_obj.best_params_
print("
Best Hyperparameters that yield the best score:
{}
".format(best_params))
pred_val =(estimator.fit(features_train_small, target_train_small)).predict(features_val_small)
 print("
Unoptimized Estimator prediction score on Validation set: \t{}".format(roc_auc_score(target_val_small, pred_val)))
 best_pred_val = best_est.predict(features_val_small)
 print("
Optimized Estimator prediction score on Validation set: \t{}".format(roc_auc_score(target_val_small, best_pred_val)))
<init_hyperparams>
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip',compression='zip')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip',compression='zip' )
if run_mode == 'train_estimator_LGBM':
params = {
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': 0.1,
'min_data_in_leaf': 30,
'num_leaves': 31,
'max_depth': -1,
'feature_fraction': 0.5,
'scale_pos_weight': 2,
'drop_rate': 0.02,
'metric': 'auc',
'num_boost_round': 200,
}
data_split = 'kfold'
if data_split == 'train_test_split':
lgb_train = lgb.Dataset(data=X_train, label=y_train)
 lgb_eval = lgb.Dataset(data=X_val, label=y_val)
 start = time()
estimator = lgb.train(
 params = params,
train_set = lgb_train,
valid_sets = lgb_eval,
early_stopping_rounds = 350,
verbose_eval = 200 
)
 end = time()
elif data_split == 'kfold':
lgb_train_cv = lgb.Dataset(data=train, label=target)
 start = time()
cv_results = lgb.cv(
 params = params,
train_set = lgb_train_cv,
nfold = 2,
early_stopping_rounds = 50,
stratified = True,
verbose_eval = 50 
)
 optimum_boost_rounds = np.argmax(cv_results['auc-mean'])
 print('Optimum boost rounds = {}'.format(optimum_boost_rounds))
print('Best LGBM CV result = {}'.format(np.max(cv_results['auc-mean'])))
 estimator = lgb.train(
 params = params,
train_set = lgb_train_cv,
num_boost_round = optimum_boost_rounds,
verbose_eval = 50 
)
 end = time()
print("
Estimator fit time: {} seconds".format(int(round(end - start))))
lgb.plot_importance(estimator, figsize=(12, 12), max_num_features=30);<find_best_params>
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()
print(estimator )<prepare_x_and_y>
max_features=100000
maxlen=150
embed_size=300
if run_mode == 'LGBM_KFold':
folds = KFold(n_splits=10, shuffle=True, random_state=1024)
 oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance = pd.DataFrame()
feats = train.columns
start = time()
for n_fold,(train_index, valid_index)in enumerate(folds.split(train, target)) :
train_x, train_y = train.iloc[train_index], target.iloc[train_index]
valid_x, valid_y = train.iloc[valid_index], target.iloc[valid_index]
clf = lgb.LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1, 
)
 clf.fit(
 train_x,
train_y,
eval_set = [(valid_x, valid_y)],
eval_metric = 'auc',
verbose = 200,
early_stopping_rounds = 500, 
)
 oof_preds[valid_index] = clf.predict_proba(valid_x, num_iterations=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test, num_iterations=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance = pd.DataFrame()
fold_importance['feature'] = feats
fold_importance['importance'] = clf.feature_importances_
fold_importance['fold'] = n_fold + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 print('Fold {:02d} AUC: {:.6f}'.format(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_index])))
 end = time()
print("
Estimator fit time: {} seconds".format(int(round(end - start))))
print('Full AUC score: {:.6f}'.format(roc_auc_score(target, oof_preds)) )<save_to_csv>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
if run_mode == 'LGBM_KFold':
submission = pd.DataFrame()
submission['SK_ID_CURR'] = test_id
submission['TARGET'] = sub_preds
submission.to_csv('LGBM_SKFold.csv', index=False )<choose_model_class>
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )
if run_mode == 'train_estimator_RFR':
estimator = RandomForestRegressor(n_estimators=125,
max_features=0.2,
min_samples_split=2,
min_samples_leaf=75,
n_jobs=-1,
random_state=42,
verbose=0)
 print("
Preparing to train the following estimator:
{}".format(estimator))
start = time()
estimator.fit(X_train, y_train)
 end = time()
print("
Estimator fit time: {} seconds".format(int(round(end - start))))
pred_val = estimator.predict(X_val)
 print("
Estimator prediction score on Validation set: \t{}".format(roc_auc_score(y_val, pred_val)))
 fi = pd.DataFrame()
fi['feature'] = X_train.columns
fi['importance'] = estimator.feature_importances_
display(fi.sort_values(by=['importance'], ascending=False ).head(10)) 
<save_to_csv>
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
if 'train_estimator_' in run_mode:
pred_test = estimator.predict(test)
 print("
Preparing prediction for submission.")
 submission = pd.DataFrame()
submission['SK_ID_CURR'] = test_id
submission['TARGET'] = pred_test
submission.head()
file_name = run_mode.split('train_estimator_')[1] + '.csv'
submission.to_csv(file_name, index=False )<set_options>
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
warnings.simplefilter(action='ignore', category=FutureWarning)
 py.init_notebook_mode(connected=True)
 %matplotlib inline
cf.go_offline() 
<categorify>
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns<define_variables>
batch_size = 128
epochs = 4
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233 )
num_rows = None
nan_as_category = True<train_model>
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model.predict(x_test,batch_size=1024,verbose=1 )
print("Start Train Test................." )<load_from_csv>
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip',compression='zip')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )