Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
%%capture
!pip install -U scikit-learn
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_pred
sub.to_csv('stacked_1.csv', index=False )<set_options>
%%capture
!pip install ktrain
__author__ = 'Tilii: https://kaggle.com/tilii7'
warnings.filterwarnings('ignore')
<prepare_x_and_y>
import numpy as np
import pandas as pd
import tensorflow as tf
import ktrain
import plotnine
import seaborn as sns
from ktrain import text
from transformers import *
from plotnine import *
from plotnine.options import figure_size
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import roc_curve, auc, accuracy_score, f1_score
from pylab import rcParams
def timer(start_time=None):
if not start_time:
start_time = datetime.now()
return start_time
elif start_time:
thour, temp_sec = divmod(( datetime.now() - start_time ).total_seconds() , 3600)
 tmin, tsec = divmod(temp_sec, 60)
 print('
Time taken: %i hours %i minutes and %s seconds.' %(thour, tmin, round(tsec, 2)))
 train_df = pd.read_csv('.. /input/train.csv', dtype={'id': np.int32, 'target': np.int8})
 Y = train_df['target'].values
X = train_df.drop(['target', 'id'], axis=1)
 test_df = pd.read_csv('.. /input/test.csv', dtype={'id': np.int32})
 test = test_df.drop(['id'], axis=1)
<define_search_space>
input_dir = '/kaggle/input/nlp-getting-started/'
checkpoint_dir = '/kaggle/input/checkpoint-nlp-disaster/'
output_dir = '/kaggle/working/'
df_fit = pd.read_csv(input_dir + 'train.csv')
 df_sub = pd.read_csv(input_dir + 'test.csv')
 df_fit.head()
params = {
'min_child_weight': [1, 5, 10],
'gamma': [0.5, 1, 1.5, 2, 5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'max_depth': [3, 4, 5]
} 
<choose_model_class>
sss = StratifiedShuffleSplit(n_splits=1, random_state=42)
 train_index, valid_index = next(sss.split(df_fit, df_fit['target']))
df_trn = df_fit.iloc[train_index, :]
df_tst = df_fit.iloc[valid_index, :]
xgb = XGBClassifier(learning_rate=0.02, n_estimators=600, objective='binary:logistic',
silent=True, nthread=1)
<train_on_grid>
benchmark_dict = dict()
model_list = []
if len(model_list)> 0:
nrow_trn, nrow_tst, nrow_sub = df_trn.shape[0], df_tst.shape[0], df_sub.shape[0]
dict_benchmark = {'Model': [], 'Type': [], 'Id': [], 'Prediction': [], 'Target': []}
dict_submission = {'Model': [], 'Id': [], 'Proba': []}
for curr_model in model_list:
curr_benchmark = dict()
t = text.Transformer(curr_model, maxlen=140, classes=[0, 1])
 trn = t.preprocess_train(df_trn['text'].tolist() , df_trn['target'].tolist())
 val = t.preprocess_train(df_tst['text'].tolist() , df_tst['target'].tolist())
 model = t.get_classifier()
learner = ktrain.get_learner(model, train_data=trn, val_data=val, batch_size=16)
 learner.fit_onecycle(2e-6, 3)
 predictor = ktrain.get_predictor(model, t)
 curr_predict_trn = predictor.predict_proba(df_trn['text'].tolist())[:, 1].tolist()
curr_predict_tst = predictor.predict_proba(df_tst['text'].tolist())[:, 1].tolist()
curr_predict_sub = predictor.predict_proba(df_sub['text'].tolist())[:, 1].tolist()
dict_benchmark['Model'] +=(nrow_trn + nrow_tst)* [curr_model]
dict_benchmark['Type'] +=(nrow_trn * ['Train'])+(nrow_tst * ['Test'])
 dict_benchmark['Id'] += df_trn['id'].tolist() + df_tst['id'].tolist()
dict_benchmark['Prediction'] += curr_predict_trn + curr_predict_tst
dict_benchmark['Target'] += df_trn['target'].tolist() + df_tst['target'].tolist()
dict_submission['Model'] += nrow_sub * [curr_model]
dict_submission['Id'] += df_sub['id'].tolist()
dict_submission['Proba'] += curr_predict_sub
df_benchmark, df_submission = pd.DataFrame(dict_benchmark), pd.DataFrame(dict_submission)
 df_benchmark.to_csv(output_dir + 'benchmark.csv'), df_submission.to_csv('submission.csv')
 else:
benchmark_list, submission_list = [], []
for model_name in ['albert', 'bert', 'roberta', 'xlnet']:
benchmark_list.append(pd.read_csv(checkpoint_dir + 'benchmark_' + model_name + '.csv'))
submission_list.append(pd.read_csv(checkpoint_dir + 'submission_' + model_name + '.csv'))
df_benchmark = pd.concat(benchmark_list, ignore_index=True)
 df_submission = pd.concat(submission_list, ignore_index=True )
folds = 3
param_comb = 5
skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 1001)
 random_search = RandomizedSearchCV(xgb, param_distributions=params, n_iter=param_comb, scoring='roc_auc', n_jobs=4, cv=skf.split(X,Y), verbose=3, random_state=1001)
 start_time = timer(None)
 random_search.fit(X, Y)
 timer(start_time)
<save_to_csv>
def get_df_roc(df_in, models_list=None):
df_roc = pd.DataFrame({'Model': [], 'FalsePositiveRate': [], 'TruePositiveRate': []})
 if models_list is None:
models_list = set(df_in['Model'].tolist())
 for curr_model in models_list:
fpr_list, tpr_list, _ = roc_curve(df_in.loc[df_in['Model'] == curr_model, :]['Target'].tolist() ,
df_in.loc[df_in['Model'] == curr_model, :]['Prediction'].tolist())
 model_list = [curr_model] * len(fpr_list)
 df2append = pd.DataFrame({
'Model': model_list,
'FalsePositiveRate': fpr_list,
'TruePositiveRate': tpr_list
})
 df_roc = pd.concat([df_roc, df2append], ignore_index=True)
 return df_roc
df_roc = get_df_roc(df_benchmark)
 df_roc.head()
print('
All results:')
 print(random_search.cv_results_)
 print('
Best estimator:')
 print(random_search.best_estimator_)
 print('
Best normalized gini score for %d-fold search with %d parameter combinations:' %(folds, param_comb))
print(random_search.best_score_ * 2 - 1)
 print('
Best hyperparameters:')
 print(random_search.best_params_)
 results = pd.DataFrame(random_search.cv_results_)
 results.to_csv('xgb-random-grid-search-results-01.csv', index=False)
<save_to_csv>
def get_benchmark_scores(df_in, models_list=set(df_benchmark['Model'].tolist())) :
dict_metrics = dict(Model=[], Metric=[], Value=[])
 for curr_model in models_list:
for metric in ['Accuracy', 'AUC', 'F1-Score']:
filter_cond = df_in['Model'] == curr_model
filter_cond &= df_in['Type'] == 'Test'
sorted_df = df_in.loc[filter_cond, ['Id', 'Prediction', 'Target']].sort_values(by='Id')
 predictions = sorted_df['Prediction'].tolist()
targets = sorted_df['Target'].tolist()
if metric in ['Accuracy', 'F1-Score']:
predictions = [1 if X >= 0.5 else 0 for X in predictions]
if metric == 'Accuracy':
score = accuracy_score(targets, predictions)
 elif metric == 'F1-Score':
score = accuracy_score(targets, predictions)
 elif metric == 'AUC':
fpr, tpr, _ = roc_curve(targets, predictions)
 score = auc(fpr, tpr)
 dict_metrics['Model'].append(curr_model)
 dict_metrics['Metric'].append(metric)
 dict_metrics['Value'].append(score)
 return pd.DataFrame(dict_metrics)
 df_metrics = get_benchmark_scores(df_benchmark)
 df_metrics.head()
y_test = random_search.predict_proba(test)
 results_df = pd.DataFrame(data={'id':test_df['id'], 'target':y_test[:,1]})
 results_df.to_csv('submission-random-grid-search-xgb-porto-01.csv', index=False)
<set_options>
dict_corr = dict(Model1=[], Model2=[], Correlation=[])
 models_list = set(df_benchmark['Model'].tolist())
 for model1 in models_list:
for model2 in models_list:
dict_corr['Model1'].append(model1)
 dict_corr['Model2'].append(model2)
 filter_cond1 = df_benchmark['Model'] == model1
filter_cond1 &= df_benchmark['Type'] == 'Test'
filter_cond2 = df_benchmark['Model'] == model2
filter_cond2 &= df_benchmark['Type'] == 'Test'
pred1 = df_benchmark.loc[filter_cond1]['Prediction'].tolist()
pred2 = df_benchmark.loc[filter_cond2]['Prediction'].tolist()
dict_corr['Correlation'].append(np.corrcoef(np.array(pred1), np.array(pred2)) [0, 1])
 dict_model_bias = {
'albert-base-v2': 'ALBERT',
'bert-base-uncased': 'BERT',
'roberta-base': 'ROBERTA',
'xlnet-large-cased': 'XLNET'
}
df_corr = pd.DataFrame(dict_corr)
 df_corr['Model1'] = df_corr['Model1'].apply(lambda X: dict_model_bias[X])
 df_corr['Model2'] = df_corr['Model2'].apply(lambda X: dict_model_bias[X])
 df_corr.head()
%matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )<load_from_csv>
def get_df_ensemble(df_in, name_new_model):
df_ensemble = df_in.groupby('Id' ).mean()
df_ensemble['Id'] = df_ensemble.index
train_ids = list(set(df_in[df_in['Type'] == 'Train']['Id'].tolist()))
test_ids = list(set(df_in[df_in['Type'] == 'Test']['Id'].tolist()))
df_ensemble['Id'] = df_ensemble.index
df_ensemble['Model'] = name_new_model
df_ensemble['Type'] = 'NA'
df_ensemble.loc[df_ensemble['Id'].isin(train_ids), 'Type'] = 'Train'
df_ensemble.loc[df_ensemble['Id'].isin(test_ids), 'Type'] = 'Test'
df_ensemble = df_ensemble.loc[:, ['Model', 'Type', 'Id', 'Prediction', 'Target']]
return df_ensemble
df_ensemble_all = get_df_ensemble(df_benchmark, 'ensemble_all')
 best_models = ['bert-base-uncased', 'roberta-base']
df_ensemble_best = get_df_ensemble(df_benchmark.loc[df_benchmark['Model'].isin(best_models), :], 'ensemble_best')
 df_benchmark_improved = pd.concat([df_benchmark, df_ensemble_all, df_ensemble_best], ignore_index=True)
 plot_benchmark_scores(df_benchmark_improved, models_list=best_models + ['ensemble_best', 'ensemble_all'] )
train = pd.read_csv('.. /input/train.csv')
 train.head()<load_from_csv>
df_submission_ensemble_all = df_submission.groupby('Id' ).mean()
df_submission_ensemble_all['Id'] = df_submission_ensemble_all.index
df_submission_ensemble_all['Model'] = 'ensemble_all'
df_submission_ensemble_all = df_submission_ensemble_all.loc[:, ['Model', 'Id', 'Proba']]
df_submission_ensemble_best = df_submission.loc[df_submission['Model'].isin(best_models), :].groupby('Id' ).mean()
df_submission_ensemble_best['Id'] = df_submission_ensemble_best.index
df_submission_ensemble_best['Model'] = 'ensemble_best'
df_submission_ensemble_best = df_submission_ensemble_best.loc[:, ['Model', 'Id', 'Proba']]
df_submission_enhanced = pd.concat([df_submission, df_submission_ensemble_all, df_submission_ensemble_best], ignore_index=True)
 df_submission_enhanced.tail()
test = pd.read_csv('.. /input/test.csv')
 test.tail()<define_variables>
def export_model(model_name):
df_export = df_submission_enhanced.loc[df_submission_enhanced['Model'] == model_name, :]
df_export.loc[:, 'Target'] = df_export['Proba'].apply(lambda X: 1 if X >= 0.5 else 0)
 df_export = df_export.loc[:, ['Id', 'Target']]
df_export.rename({'Id': 'id', 'Target': 'target'}, inplace=True)
 df_export.to_csv(output_dir + model_name + '.csv', index=False )
<count_missing_values><EOS> 
export_model('ensemble_all')
 export_model('ensemble_best')
 export_model('bert-base-uncased')
 export_model('roberta-base' )