Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<save_to_csv>
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)
submission.time_to_failure = rf_sub
submission.to_csv('submission.csv',index=True )<load_from_csv>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
df= pd.read_csv('.. /input/best-score-ever/best_score.csv' )<save_to_csv>
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 y = train_df[list_classes].values
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = y
x_test = test_df[TEXT_COLUMN].astype(str )
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission['time_to_failure'] = df.time_to_failure
submission.to_csv('submission.csv',index=False )<load_from_csv>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
train = pd.read_csv('.. /input/eq-100-features/train_100.csv')
 test = pd.read_csv('.. /input/eq-100-features/test_100.csv')
 y = pd.read_csv('.. /input/eq-100-features/y.csv' ).values.flatten()<define_variables>
%%time
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
warnings.filterwarnings('ignore')
 NUM_EVALS = 1000
N_FOLDS = 5
LGBM_MAX_LEAVES = 2**10
LGBM_MAX_DEPTH = 25
EVAL_METRIC_LGBM_REG = 'mae'
EVAL_METRIC_LGBM_CLASS = 'auc'
XGB_MAX_LEAVES = 2**12
XGB_MAX_DEPTH = 25
EVAL_METRIC_XGB_REG = 'mae'
EVAL_METRIC_XGB_CLASS = 'auc'
CB_MAX_DEPTH = 8
OBJECTIVE_CB_REG = 'MAE'
OBJECTIVE_CB_CLASS = 'Logloss'
BEST_SCORE = 0
def quick_hyperopt(data, labels, package='lgbm', num_evals=NUM_EVALS, diagnostic=False, Class=False):
if package=='lgbm':
print('Running {} rounds of LightGBM parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['max_depth',
'num_leaves',
'max_bin',
'min_data_in_leaf',
'min_data_in_bin']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['boosting']['boosting'] == 'goss':
top_rate = space_params['boosting'].get('top_rate')
 other_rate = space_params['boosting'].get('other_rate')
 top_rate = max(top_rate, 0)
 top_rate = min(top_rate, 0.5)
 other_rate = max(other_rate, 0)
 other_rate = min(other_rate, 0.5)
 space_params['top_rate'] = top_rate
space_params['other_rate'] = other_rate
subsample = space_params['boosting'].get('subsample', 1.0)
 space_params['boosting'] = space_params['boosting']['boosting']
space_params['subsample'] = subsample
if Class:
cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=True,
early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_CLASS, seed=42)
 best_loss = 1 - cv_results['auc-mean'][-1]
else:
cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=False,
early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_REG, seed=42)
 best_loss = cv_results['l1-mean'][-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = lgb.Dataset(data, labels)
 boosting_list = [{'boosting': 'gbdt',
'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting': 'goss',
'subsample': 1.0,
'top_rate': hp.uniform('top_rate', 0, 0.5),
'other_rate': hp.uniform('other_rate', 0, 0.5)}]
if Class:
metric_list = ['auc']
objective_list = ['binary', 'cross_entropy']
else:
metric_list = ['MAE', 'RMSE']
objective_list = ['huber', 'gamma', 'fair', 'tweedie']
space ={'boosting' : hp.choice('boosting', boosting_list),
'num_leaves' : hp.quniform('num_leaves', 2, LGBM_MAX_LEAVES, 1),
'max_depth': hp.quniform('max_depth', 2, LGBM_MAX_DEPTH, 1),
'max_bin': hp.quniform('max_bin', 32, 255, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 256, 1),
'min_data_in_bin': hp.quniform('min_data_in_bin', 1, 256, 1),
'min_gain_to_split' : hp.quniform('min_gain_to_split', 0.1, 5, 0.01),
'lambda_l1' : hp.uniform('lambda_l1', 0, 5),
'lambda_l2' : hp.uniform('lambda_l2', 0, 5),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) ,
'metric' : hp.choice('metric', metric_list),
'objective' : hp.choice('objective', objective_list),
'feature_fraction' : hp.quniform('feature_fraction', 0.5, 1, 0.01),
'bagging_fraction' : hp.quniform('bagging_fraction', 0.5, 1, 0.01)
 }
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['boosting'] = boosting_list[best['boosting']]['boosting']
best['metric'] = metric_list[best['metric']]
best['objective'] = objective_list[best['objective']]
for param in integer_params:
best[param] = int(best[param])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 if package=='xgb':
print('Running {} rounds of XGBoost parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['max_depth']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['tree_method']['tree_method'] == 'hist':
max_bin = space_params['tree_method'].get('max_bin')
 space_params['max_bin'] = int(max_bin)
 if space_params['tree_method']['grow_policy']['grow_policy']['grow_policy'] == 'depthwise':
grow_policy = space_params['tree_method'].get('grow_policy' ).get('grow_policy' ).get('grow_policy')
 space_params['grow_policy'] = grow_policy
space_params['tree_method'] = 'hist'
else:
max_leaves = space_params['tree_method']['grow_policy']['grow_policy'].get('max_leaves')
 space_params['grow_policy'] = 'lossguide'
space_params['max_leaves'] = int(max_leaves)
 space_params['tree_method'] = 'hist'
else:
space_params['tree_method'] = space_params['tree_method'].get('tree_method')
 cv_results = xgb.cv(space_params, train, nfold=N_FOLDS, metrics=[EVAL_METRIC_XGB_REG],
early_stopping_rounds=100, stratified=False, seed=42)
 best_loss = cv_results['test-mae-mean'].iloc[-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = xgb.DMatrix(data, labels)
 boosting_list = ['gbtree', 'gblinear']
metric_list = ['MAE', 'RMSE']
tree_method = [{'tree_method' : 'exact'},
{'tree_method' : 'approx'},
{'tree_method' : 'hist',
'max_bin': hp.quniform('max_bin', 2**3, 2**7, 1),
'grow_policy' : {'grow_policy': {'grow_policy':'depthwise'},
'grow_policy' : {'grow_policy':'lossguide',
'max_leaves': hp.quniform('max_leaves', 32, XGB_MAX_LEAVES, 1)}}}]
objective_list_reg = ['reg:linear', 'reg:gamma', 'reg:tweedie']
objective_list_class = ['reg:logistic', 'binary:logistic']
objective_list = objective_list_reg
space ={'boosting' : hp.choice('boosting', boosting_list),
'tree_method' : hp.choice('tree_method', tree_method),
'max_depth': hp.quniform('max_depth', 2, XGB_MAX_DEPTH, 1),
'reg_alpha' : hp.uniform('reg_alpha', 0, 5),
'reg_lambda' : hp.uniform('reg_lambda', 0, 5),
'min_child_weight' : hp.uniform('min_child_weight', 0, 5),
'gamma' : hp.uniform('gamma', 0, 5),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) ,
'eval_metric' : hp.choice('eval_metric', metric_list),
'objective' : hp.choice('objective', objective_list),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1, 0.01),
'colsample_bynode' : hp.quniform('colsample_bynode', 0.1, 1, 0.01),
'colsample_bylevel' : hp.quniform('colsample_bylevel', 0.1, 1, 0.01),
'subsample' : hp.quniform('subsample', 0.5, 1, 0.05),
'nthread' : -1
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['tree_method'] = tree_method[best['tree_method']]['tree_method']
best['boosting'] = boosting_list[best['boosting']]
best['eval_metric'] = metric_list[best['eval_metric']]
best['objective'] = objective_list[best['objective']]
for param in integer_params:
best[param] = int(best[param])
 if 'max_leaves' in best:
best['max_leaves'] = int(best['max_leaves'])
 if 'max_bin' in best:
best['max_bin'] = int(best['max_bin'])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 if package=='cb':
print('Running {} rounds of CatBoost parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['depth',
'min_data_in_leaf',
'max_bin']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['bootstrap_type']['bootstrap_type'] == 'Bayesian':
bagging_temp = space_params['bootstrap_type'].get('bagging_temperature')
 space_params['bagging_temperature'] = bagging_temp
if space_params['grow_policy']['grow_policy'] == 'LossGuide':
max_leaves = space_params['grow_policy'].get('max_leaves')
 space_params['max_leaves'] = int(max_leaves)
 space_params['bootstrap_type'] = space_params['bootstrap_type']['bootstrap_type']
space_params['grow_policy'] = space_params['grow_policy']['grow_policy']
space_params['random_strength'] = max(space_params['random_strength'], 0)
 space_params['fold_len_multiplier'] = max(space_params['fold_len_multiplier'], 1)
 cv_results = cb.cv(train, space_params, fold_count=N_FOLDS,
early_stopping_rounds=25, stratified=False, partition_random_seed=42)
 best_loss = cv_results['test-MAE-mean'].iloc[-1]
return{'loss':best_loss, 'status': STATUS_OK}
train = cb.Pool(data, labels.astype('float32'))
bootstrap_type = [{'bootstrap_type':'Poisson'},
{'bootstrap_type':'Bayesian',
'bagging_temperature' : hp.loguniform('bagging_temperature', np.log(1), np.log(50)) },
{'bootstrap_type':'Bernoulli'}]
LEB = ['No', 'AnyImprovement', 'Armijo']
grow_policy = [{'grow_policy':'SymmetricTree'},
{'grow_policy':'Depthwise'},
{'grow_policy':'Lossguide',
'max_leaves': hp.quniform('max_leaves', 2, 32, 1)}]
eval_metric_list_reg = ['MAE', 'RMSE', 'Poisson']
eval_metric_list_class = ['Logloss', 'AUC', 'F1']
eval_metric_list = eval_metric_list_reg
space ={'depth': hp.quniform('depth', 2, CB_MAX_DEPTH, 1),
'max_bin' : hp.quniform('max_bin', 1, 32, 1),
'l2_leaf_reg' : hp.uniform('l2_leaf_reg', 0, 5),
'min_data_in_leaf' : hp.quniform('min_data_in_leaf', 1, 50, 1),
'random_strength' : hp.loguniform('random_strength', np.log(0.005), np.log(5)) ,
'bootstrap_type' : hp.choice('bootstrap_type', bootstrap_type),
'learning_rate' : hp.uniform('learning_rate', 0.05, 0.25),
'eval_metric' : hp.choice('eval_metric', eval_metric_list),
'objective' : OBJECTIVE_CB_REG,
'leaf_estimation_backtracking' : hp.choice('leaf_estimation_backtracking', LEB),
'grow_policy': hp.choice('grow_policy', grow_policy),
'fold_len_multiplier' : hp.loguniform('fold_len_multiplier', np.log(1.01), np.log(2.5)) ,
'od_type' : 'Iter',
'od_wait' : 25,
'task_type' : 'GPU',
'verbose' : 0
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['bootstrap_type'] = bootstrap_type[best['bootstrap_type']]['bootstrap_type']
best['grow_policy'] = grow_policy[best['grow_policy']]['grow_policy']
best['eval_metric'] = eval_metric_list[best['eval_metric']]
best['leaf_estimation_backtracking'] = LEB[best['leaf_estimation_backtracking']]
for param in integer_params:
best[param] = int(best[param])
 if 'max_leaves' in best:
best['max_leaves'] = int(best['max_leaves'])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 else:
print('Package not recognised.Please use "lgbm" for LightGBM, "xgb" for XGBoost or "cb" for CatBoost.' )<train_model>
%%time
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )
lgbm_params = quick_hyperopt(train, y, 'lgbm', 150 )<split>
X_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train, test_size = 0.1)
def quick_kfold_imp(X, y, test=None, params=None, n_fold=5, random_state=42):
MAE = 0
folds = KFold(n_splits=n_fold, shuffle=True, random_state=random_state)
 test_preds = np.zeros(len(test))
imp_s = np.zeros(X.shape[1])
 imp_g = np.zeros(X.shape[1])
 if type(y)is not np.ndarray:
y = y.values.flatten()
for train_idx, valid_idx in folds.split(y):
X_train, X_valid = X.iloc[train_idx, :], X.iloc[valid_idx, :]
y_train, y_valid = y[train_idx], y[valid_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1, eval_metric='mae', importance_type='split')
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=0, early_stopping_rounds=200)
 val_preds = model.predict(X_valid)
 imp_s += model.feature_importances_/n_fold
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1, eval_metric='mae', importance_type='gain')
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=0, early_stopping_rounds=200)
 val_preds = model.predict(X_valid)
 MAE += mean_absolute_error(y_valid, val_preds)/n_fold
imp_g += model.feature_importances_/n_fold
test_preds += model.predict(test)/n_fold
print('OOF MAE: {}'.format(MAE))
return(imp_s, imp_g, test_preds )<init_hyperparams>
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 X_train,
Y_train,
validation_data =(X_valid, Y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 val_preds_3 = model.predict(X_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(Y_valid[:,i], val_preds_3[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5')
<save_to_csv><EOS> 
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )