Kaggle competition: Natural Language Processing with Disaster Tweets
max_alpha_ix = np.argmax(np.vstack(( mean_train_auc_roc, mean_val_auc_roc)).mean(axis=0))
max_mean_cv_score = np.max(np.vstack(( mean_train_auc_roc, mean_val_auc_roc)).mean(axis=0))
alpha = alpha_values[max_alpha_ix]
print("alpha: ", alpha, "max_mean, cv_score: ", max_mean_cv_score )<train_model>
plt.style.use('ggplot')
lr = ElasticNet(alpha=alpha, l1_ratio=0.75, normalize=False, max_iter=1000, precompute=True)
 lr.fit(X_train, y_train)
 test_predictions = lr.predict(test_df.values )<save_to_csv>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 display(train.head())
 print(len(train))
display(test.head())
 print(len(test))
submission = pd.DataFrame({
'id': np.arange(250,20000),
'target': test_predictions
})
 submission.to_csv("submission.csv", index=False )<load_from_csv>
def create_corpus(target):
corpus = []
for x in train[train["target"] == target]["text"].str.split() :
print(x)
 for i in x:
corpus.append(i)
 return corpus
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
from collections import defaultdict
from nltk.corpus import stopwords
from nltk.util import ngrams
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_values>
corpus = create_corpus(1)
 dictionary = defaultdict(int)
 special_char = string.punctuation
for i in corpus:
if i in special_char:
dictionary[i] +=1
train.target.value_counts()<count_values>
corpus = create_corpus(0)
 dictionary = defaultdict(int)
 special_char = string.punctuation
for i in corpus:
if i in special_char:
dictionary[i] +=1
train.target.value_counts() / len(train )<count_missing_values>
from collections import Counter
train.isna().sum().sum()<count_missing_values>
counter = Counter(corpus)
 most = counter.most_common()
x = []
y = []
for word, count in most[:40]:
if word not in stop:
x.append(word)
 y.append(count )
test.isna().sum().sum()<remove_duplicates>
df = pd.concat([train, test])
 df.shape
s = corr.unstack().drop_duplicates()
so = s.sort_values(kind="quicksort")
 so = so.drop_duplicates()<sort_values>
import re
s["target"].sort_values() [:10]<sort_values>
def remove_url(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'', text )
s["target"].sort_values(ascending=False)[:10]<sort_values>
df["text"] = df["text"].apply(lambda x: remove_url(x))
print("Top most highly positive correlated features:")
 print(so[(so<1)&(so>0.2)].sort_values(ascending=False))
print()
print("Top most highly megative correlated features:")
 print(so[(so < - 0.2)] )<set_options>
def remove_html(text):
html = re.compile(r'<.*?>')
 return html.sub(r'', text )
fs_params = dict()
fs_params["descriptives"] = False
fs_params["standardization"] = False
fs_params["percentiles"] = False
fs_params["squared"] = False
fs_params["frequency"] = False
gc.collect() ;
turn = 0
for df in [test, train]:
if turn == 0:
print("Train set")
 turn = 1
else:
print("Test set")
 if(fs_params["descriptives"] == True):
print('\t*descriptive statistics Feature Engineering:')
 df['sum'] = df[numerical_features].sum(axis=1)
 df['min'] = df[numerical_features].min(axis=1)
 df['max'] = df[numerical_features].max(axis=1)
 df['mean'] = df[numerical_features].mean(axis=1)
 df['std'] = df[numerical_features].std(axis=1)
 df['skew'] = df[numerical_features].skew(axis=1)
 df['kurt'] = df[numerical_features].kurtosis(axis=1)
 df['med'] = df[numerical_features].median(axis=1)
 print('\t*descriptive statistics Feature Engineering done!')
 if(fs_params["standardization"] == True):
print('\t*Standardizing the data:')
 scaler = StandardScaler()
df[numerical_features] = scaler.fit_transform(df[numerical_features])
 print('\t*Data Standardized!')
 if(fs_params["percentiles"] == True):
print('\t*percentiles Feature Engineering:')
 perc_list = [1,2,5,10,25,50,60,75,80,85,95,99]
for i in perc_list:
df['perc_'+str(i)] = df[numerical_features].apply(lambda x: np.percentile(x, i), axis=1)
 print('\t*Done percentiles Feature Engineering!')
 if(fs_params["squared"] == True):
print('\t*Loading Squared data:')
 for i in numerical_features:
df['var_sq_'+str(i)] = np.square(df[str(i)])
 print('\t*Done squaring!')
 if(fs_params["frequency"] == True):
print('\t*Loading frequency:')
 for var in numerical_features:
hist, bin_edges = np.histogram(df[var], bins=1000, density=True)
 df['hist_'+var] = [ hist[np.searchsorted(bin_edges,ele)-1] for ele in df[var]]
print('\t*Done Loading frequency!')
 gc.collect() ;<prepare_x_and_y>
df["text"] = df["text"].apply(lambda x: remove_html(x))
y = train['target']
X = train.drop(['target', "id"], axis=1)
 xtest = test.drop("id", axis="columns")
 std_scaler_flag = False
robust_scaler_flag = False
min_max_scalar_flag = False
if std_scaler_flag == True:
print("StandardScaler!")
 sc0 = StandardScaler()
sc0.fit(X)
 X = sc0.transform(X)
 X = pd.DataFrame(X, columns=train.drop(['target', "id"], axis=1 ).columns)
 xtest = sc0.transform(xtest)
 xtest = pd.DataFrame(xtest, columns = test.drop("id", axis="columns" ).columns)
 print(X.head(6))
else:
print("No scaling at all!" )<create_dataframe>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 return emoji_pattern.sub(r'', text )
clf_stats_df = pd.DataFrame(columns=["clf_name", "F1-score", "auc-score"] )<choose_model_class>
df["text"] = df["text"].apply(lambda x: remove_emoji(x))
def xgboost_all_purpose(X, y, type_of_training, name, num_of_folds=3, params=None, in_folds_sampling = False, max_early_stopping = 100):
global clf_stats_df
if params is None:
params = dict()
params["learning_rate"] = 0.1
params["n_estimators"] = 500
params["max_depth"] = 3
params["min_child_weight"] = 1
params["gamma"] = 0
params["subsample"] = 1
params["colsample_bytree"] = 1
params["colsample_bylevel"] = 1
params["reg_alpha"] = 0
params["reg_lambda"] = 1
params["scale_pos_weight"] = np.round(y.value_counts() [0] / y.value_counts() [1],3)
 params["max_delta_step"] = 1
print("params", params)
 print("max_early_stopping:", max_early_stopping)
 if type_of_training == "baseline":
print("baseline")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test = np.zeros(len(test))
num_fold = 0
folds = StratifiedKFold(n_splits=num_of_folds, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_stra_xgb = xgb.XGBClassifier(learning_rate=params["learning_rate"],
n_estimators=params["n_estimators"],
max_depth=params["max_depth"],
min_child_weight=params["min_child_weight"],
gamma=params["gamma"],
subsample=params["subsample"],
colsample_bytree=params["colsample_bytree"],
colsample_bylevel=params["colsample_bylevel"],
objective= 'binary:logistic',
nthread=-1,
scale_pos_weight=params["scale_pos_weight"],
reg_alpha = params["reg_alpha"],
reg_lambda = params["reg_lambda"],
max_delta_step = params["max_delta_step"],
seed=42)
 clf_stra_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=max_early_stopping, eval_metric='auc', verbose=100)
 predictions = clf_stra_xgb.predict(xvalid)
 predictions_probas = clf_stra_xgb.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test += clf_stra_xgb.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(12, 38)})
 xgb.plot_importance(clf_stra_xgb, title='Feature importance', xlabel='F score', ylabel='Features')
 clf_stats_df = clf_stats_df.append({"clf_name": name,
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
gc.collect() ;
return clf_stra_xgb, predictions_test
elif type_of_training == "train_test_equal":
print("train_test_equal_flag")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test = np.zeros(len(test))
num_fold = 0
print("resample xtrain set to be equal to the size of the test set")
 temp_df = pd.DataFrame(xtrain, columns=xtrain.columns)
 temp_df["target"] = ytrain
temp_df = temp_df.sample(frac = len(test)/ len(xtrain), replace = True)
 ytrain = temp_df['target']
xtrain = temp_df.drop(['target'], axis=1)
 del temp_df;
print(xtrain.shape, test.shape)
 folds = StratifiedKFold(n_splits=num_of_folds, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_stra_xgb = xgb.XGBClassifier(learning_rate=params["learning_rate"],
n_estimators=params["n_estimators"],
max_depth=params["max_depth"],
min_child_weight=params["min_child_weight"],
gamma=params["gamma"],
subsample=params["subsample"],
colsample_bytree=params["colsample_bytree"],
colsample_bylevel=params["colsample_bylevel"],
objective= 'binary:logistic',
nthread=-1,
scale_pos_weight=params["scale_pos_weight"],
reg_alpha = params["reg_alpha"],
reg_lambda = params["reg_lambda"],
max_delta_step = params["max_delta_step"],
seed=42)
 clf_stra_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=max_early_stopping, eval_metric='auc', verbose=100)
 predictions = clf_stra_xgb.predict(xvalid)
 predictions_probas = clf_stra_xgb.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test += clf_stra_xgb.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(12, 38)})
 xgb.plot_importance(clf_stra_xgb, title='Feature importance', xlabel='F score', ylabel='Features')
 clf_stats_df = clf_stats_df.append({"clf_name": name,
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
gc.collect() ;
return clf_stra_xgb, predictions_test
else:
print("Please specify for the argument 'type_of_training'one of the following parameters:(baseline, oversampling, smote, undersampling, augmentation_by_fraction)" )<train_on_grid>
def remove_punctuation(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table )
type_of_training = "baseline"
num_of_folds = 4
in_folds_sampling = False
clf_xgb, predictions_test_xgb = xgboost_all_purpose(X,y, num_of_folds = num_of_folds,
type_of_training =type_of_training,
in_folds_sampling = in_folds_sampling,
max_early_stopping = 200,
name="clf_xgb")
 del clf_xgb<split>
df["text"] = df["text"].apply(lambda x: remove_punctuation(x))
gc.collect()
feature_selection_flag = True
if feature_selection_flag == True:
gc.collect() ;
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, test_size=0.3, random_state=42)
 fs_model = xgb.XGBClassifier(nthread = -1,
objective = 'binary:logistic',
eval_metric = 'auc',
silent=1,
tree_method='auto',
max_depth = 3,
scale_pos_weight = np.round(y.value_counts() [0] / y.value_counts() [1],3))
fs_model.fit(xtrain, ytrain)
 perm = PermutationImportance(fs_model, random_state=42 ).fit(xvalid, yvalid )<load_pretrained>
!pip install pyspellchecker
eli5.show_weights(perm, feature_names = xvalid.columns.tolist() , top=100 )<features_selection>
from spellchecker import SpellChecker
if feature_selection_flag == True:
max_selected_features = 50
sel = SelectFromModel(perm, max_features = max_selected_features, prefit=True)
 feature_idx = sel.get_support()
selected_feature_names = X.columns[feature_idx]
X_fs_50 = X[selected_feature_names]
print(X_fs_50.shape)
 max_selected_features = 25
sel = SelectFromModel(perm, max_features = max_selected_features, prefit=True)
 feature_idx = sel.get_support()
selected_feature_names = X.columns[feature_idx]
X_fs_25 = X[selected_feature_names]
print(X_fs_25.shape)
 max_selected_features = 15
sel = SelectFromModel(perm, max_features = max_selected_features, prefit=True)
 feature_idx = sel.get_support()
selected_feature_names = X.columns[feature_idx]
X_fs_15 = X[selected_feature_names]
print(X_fs_15.shape )<compute_train_metric>
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text )
fs_clf_xgb, predictions_test_fs_50_xgb = xgboost_all_purpose(X_fs_50,
y,
type_of_training =type_of_training,
num_of_folds = num_of_folds,
max_early_stopping= 100,
name="fs_clf_50_xgb" )<groupby>
from tqdm import tqdm
from nltk.tokenize import word_tokenize
fs_clf_xgb, predictions_test_fs_25_xgb = xgboost_all_purpose(X_fs_25,
y,
type_of_training =type_of_training,
num_of_folds = num_of_folds,
max_early_stopping= 100,
name="fs_clf_25_xgb" )<groupby>
def create_corpus(df):
corpus = []
for tweet in tqdm(df["text"]):
words = [word.lower() for word in word_tokenize(tweet)if \ 
(( word.isalpha() == 1)&(word not in stop)) ]
corpus.append(words)
 return corpus
fs_clf_xgb, predictions_test_fs_15_xgb = xgboost_all_purpose(X_fs_15,
y,
type_of_training =type_of_training,
num_of_folds = num_of_folds,
max_early_stopping= 100,
name="fs_clf_15_xgb" )<split>
corpus = create_corpus(df )
as_it_is = True
smote_flag = False
adasyn_flag = False
oversampling_flag = False
augment_flag = False
def prepare_for_tuning(X, y, type_of_training):
if type_of_training == "baseline":
print("no resampling at all!")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 elif type_of_training == "train_test_equal":
print("train_test_equal")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 print("resample xtrain set to be equal to the size of the test set")
 temp_df = pd.DataFrame(xtrain, columns=xtrain.columns)
 temp_df["target"] = ytrain
temp_df = temp_df.sample(frac = len(test)/ len(xtrain), replace = True)
 ytrain = temp_df['target']
xtrain = temp_df.drop(['target'], axis=1)
 del temp_df;
print(xtrain.shape, test.shape)
 elif type_of_training == "oversampling":
print("oversampling")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 ros = RandomOverSampler(random_state=42)
 X_resampled, y_resampled = ros.fit_resample(xtrain, ytrain)
 print(sorted(Counter(y_resampled ).items()))
xtrain=pd.DataFrame(X_resampled, columns = X.columns)
 ytrain = y_resampled
del X_resampled
del y_resampled
elif type_of_training == "augmentation_by_fraction":
print("augmentation")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 print("ytrain target values count before augmentation:
", sorted(Counter(ytrain ).items()))
X_y = pd.DataFrame(xtrain, columns=X.columns)
 X_y["target"] = ytrain
X_y = X_y.sample(frac=3, replace=True)
 X_y.target.value_counts()
ytrain = X_y['target']
print("ytrain target values count after oversampling:
",sorted(Counter(ytrain ).items()))
xtrain = X_y.drop(['target'], axis=1)
 del X_y
rus = RandomUnderSampler(sampling_strategy=0.33, random_state=42)
 X_resampled, y_resampled = rus.fit_resample(xtrain, ytrain)
 print("ytrain target values count after Augmentation:
",sorted(Counter(y_resampled ).items()))
xtrain=pd.DataFrame(X_resampled, columns = X.columns)
 ytrain = y_resampled
del X_resampled
del y_resampled
gc.collect() ;
return xtrain, ytrain<train_on_grid>
embedding_dict = {}
with open('.. /input/glove6b100dtxt/glove.6B.100d.txt','r')as glove:
for line in glove:
values = line.split()
word = values[0]
vectors = np.asarray(values[1:], 'float32')
 embedding_dict[word] = vectors
glove.close()
def bayesian_tuning(xtrain, ytrain):
bayes_cv_tuner = BayesSearchCV(
 estimator = xgb.XGBClassifier(
 nthread = -1,
objective = 'binary:logistic',
eval_metric = 'auc',
silent=1,
tree_method='auto' 
),
search_spaces = {
'learning_rate':(0.01, 1.0, 'log-uniform'),
'min_child_weight':(0, 10),
'n_estimators':(50, 100),
'max_depth':(0, 12),
'gamma':(1e-2, 10, 'log-uniform'),
'subsample':(0.01, 1.0, 'uniform'),
'colsample_bytree':(0.01, 1.0, 'uniform'),
'colsample_bylevel':(0.01, 1.0, 'uniform'),
'scale_pos_weight':(0.01, 1.0, 'uniform'),
'reg_lambda':(1e-1, 10, 'log-uniform'),
'reg_alpha':(1e-2, 1.0, 'log-uniform'),
'max_delta_step':(0, 10, 'uniform'),
'scale_pos_weight':(1e-2, 1, 'uniform')
 },
scoring = 'roc_auc',
cv = StratifiedKFold(
 n_splits=3,
shuffle=True,
random_state=42 
),
n_jobs = 1,
n_iter = 7,
verbose = 0,
refit = True,
random_state = 42 
)
 def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best ROC-AUC: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
result = bayes_cv_tuner.fit(xtrain, ytrain, callback = status_print)
 return result
xtrain, ytrain = prepare_for_tuning(X, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )<train_on_grid>
import tensorflow as tf
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Embedding,LSTM,Dense,SpatialDropout1D, Dropout, Input
from keras.initializers import Constant
from sklearn.model_selection import train_test_split
from keras.optimizers import Adam
from keras import regularizers
from transformers import TFAutoModel, AutoTokenizer
from keras.models import Model
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_xgb, predictions_test_tuned_xgb = xgboost_all_purpose(X,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_xgb" )<train_model>
MAX_LEN = 50
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences = tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad = pad_sequences(sequences,
maxlen = MAX_LEN,
truncating = 'post',
padding = 'post' )
xtrain, ytrain = prepare_for_tuning(X_fs_50, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )<train_on_grid>
word_index = tokenizer_obj.word_index
print('number of unique words: ', len(word_index))
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_fs_50_xgb, predictions_test_tuned_fs_50_xgb = xgboost_all_purpose(X_fs_50,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_fs_50_xgb" )<train_model>
num_words = len(word_index)+ 1
embedding_matrix = np.zeros(( num_words,100))
for word, i in tqdm(word_index.items()):
if i > num_words:
continue
embedding_vector = embedding_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
xtrain, ytrain = prepare_for_tuning(X_fs_25, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )<train_on_grid>
def encode_tweets(tokenizer, tweets, max_len):
nb_tweets = len(tweets)
 tokens = np.ones(( nb_tweets,max_len),dtype='int32')
 masks = np.zeros(( nb_tweets,max_len),dtype='int32')
 segs = np.zeros(( nb_tweets,max_len),dtype='int32')
 for k in range(nb_tweets):
tweet = tweets[k]
enc = tokenizer.encode(tweet)
 if len(enc)< max_len-2:
tokens[k,:len(enc)+2] = [0] + enc + [2]
masks[k,:len(enc)+2] = 1
else:
tokens[k,:max_len] = [0] + enc[:max_len-2] + [2]
masks[k,:max_len] = 1
return tokens,masks,segs
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_fs_25_xgb, predictions_test_tuned_fs_25_xgb = xgboost_all_purpose(X_fs_25,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_fs_25_xgb" )<train_model>
def build_model(max_len):
ids = Input(( max_len,), dtype=tf.int32)
 attention = Input(( max_len,), dtype=tf.int32)
 token = Input(( max_len,), dtype=tf.int32)
 bertweet = TFAutoModel.from_pretrained("vinai/bertweet-base")
 x,_ = bertweet(ids,attention_mask=attention,token_type_ids=token)
 out = Dense(1,activation='sigmoid' )(x[:,0,:])
 model = Model(inputs=[ids, attention, token], outputs = out)
 optimizer = Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model
model = build_model(MAX_LEN)
 model.summary()
xtrain, ytrain = prepare_for_tuning(X_fs_15, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )<train_on_grid>
optimizer = Adam(learning_rate=1e-5)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ["accuracy"] )
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_fs_15_xgb, predictions_test_tuned_fs_15_xgb = xgboost_all_purpose(X_fs_15,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_fs_15_xgb" )<save_to_csv>
test_data = tweet_pad[train.shape[0]:]
gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_xgb
submission.to_csv('clf_xgb.csv', index=False)
 if feature_selection_flag == True:
gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_fs_50_xgb
submission.to_csv('fs_50_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_fs_25_xgb
submission.to_csv('fs_25_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_fs_15_xgb
submission.to_csv('fs_15_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_xgb
submission.to_csv('tuned_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_fs_50_xgb
submission.to_csv('tuned_clf_fs_50_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_fs_25_xgb
submission.to_csv('tuned_clf_fs_25_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_fs_15_xgb
submission.to_csv('tuned_clf_fs_15_xgb.csv', index=False )<define_variables>
tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-base" )
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.1)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test = np.zeros(len(test))
num_fold = 0
num_of_folds = 10
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_logit = LogisticRegression(random_state=42,
C=0.1,
solver='liblinear',
class_weight ="balanced",
penalty = 'l1',
multi_class='ovr' ).fit(xtrain_stra, ytrain_stra)
 predictions = clf_logit.predict(xvalid)
 predictions_probas = clf_logit.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test += clf_logit.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 clf_stats_df = clf_stats_df.append({"clf_name": "clf_logit",
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()<train_model>
train_tokens, train_masks, train_segs = encode_tweets(tokenizer, train["text"].to_list() , MAX_LEN)
 train_labels = train["target"]
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.1)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test_rfe = np.zeros(len(test))
num_fold = 0
num_of_folds = 10
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_logit = LogisticRegression(random_state=42,
C=0.1,
solver='liblinear',
class_weight ="balanced",
penalty = 'l1',
multi_class='ovr')
 selector = RFE(clf_logit, 21)
 selector.fit(xtrain_stra, ytrain_stra)
 clf_logit_rfe = selector
predictions = clf_logit_rfe.predict(xvalid)
 predictions_probas = clf_logit_rfe.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test_rfe += clf_logit_rfe.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 clf_stats_df = clf_stats_df.append({"clf_name": "clf_logit",
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()<split>
ES = EarlyStopping(monitor='val_loss', mode='min', patience=10, restore_best_weights=True, verbose=1 )
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.1)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test_voting = np.zeros(len(test))
num_fold = 0
num_of_folds = 10
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
log_clf = LogisticRegression(C = 0.1, class_weight= 'balanced', penalty= 'l1', solver= 'liblinear',random_state=42)
 rand_clf = RandomForestClassifier(max_depth=2,random_state=42)
 svm_clf = SVC(gamma='auto',probability=True, random_state=42)
 extra_clf = ExtraTreesClassifier(max_depth=2,random_state=42)
 voting_clf = VotingClassifier(
 estimators = [('lr',log_clf),('rf',rand_clf),('ex',extra_clf),('sv',svm_clf)],
voting='soft')
 voting_clf.fit(xtrain_stra, ytrain_stra)
 predictions = voting_clf.predict(xvalid)
 predictions_probas = voting_clf.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test_voting += voting_clf.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 clf_stats_df = clf_stats_df.append({"clf_name": "clf_voting",
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()<save_to_csv>
hist = model.fit([train_tokens,train_masks,train_segs], train_labels,
batch_size = 32, epochs = 50,
validation_split = 0.1, callbacks= [ES] )
gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test
submission.to_csv('clf_logit.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_rfe
submission.to_csv('clf_logit_rfe.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_voting
submission.to_csv('clf_logit_voting.csv', index=False )<set_options>
submit = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
warnings.filterwarnings('ignore' )<load_from_csv>
test_tokens, test_masks, test_segs = encode_tweets(tokenizer,test["text"].to_list() , MAX_LEN )
df_train = pd.read_csv(".. /input/train.csv")
<prepare_x_and_y>
test["target"] = model.predict([test_tokens, test_masks, test_segs] ).round().astype(int)
 submission = test[["id", "target"]]
submission.to_csv("submission.csv",index=False )