Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
import numpy as np
import pandas as pd
import os
import pickle
import tensorflow as tf
from tensorflow.keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate
from tensorflow.keras.layers import LSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from tensorflow.keras.preprocessing import text, sequence
from tensorflow.keras.models import Model
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores<compute_test_metric>
CRAWL_EMBEDDING_PATH = '.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'
GLOVE_EMBEDDING_PATH = '.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'
predictions, gbm_fi = model_gbm(train_set,train_labels,test_set,test_ids,return_preds=True )<filter>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path,'rb')as f:
emb_arr = pickle.load(f)
 return emb_arr
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
predictions.loc[predictions['idhogar'] == '000a08204',:]<groupby>
sub = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sub.head()
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'] )<filter>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv' )
predictions.loc[predictions['idhogar'] == '000a08204',:]<save_to_csv>
TEXT_COLUMN = 'text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids, return_preds=False)
 submission.to_csv('gbm_baseline.csv' )<train_model>
x_train = train[TEXT_COLUMN].astype(str)
 y_train = train[TARGET_COLUMN].values
x_test = test[TEXT_COLUMN].astype(str )
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels,
test_selected, test_ids )<create_dataframe>
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE, lower=False)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
model_results = model_results.append(pd.DataFrame({'model': ["GBM", "GBM_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True)
 model_results.reset_index(drop=True )<compute_train_metric>
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids,
nfolds=10, return_preds=False )<save_to_csv>
MAX_LEN = 300
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
submission.to_csv('gbm_10fold.csv', index = False )<train_model>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids,
nfolds=10 )<save_to_csv>
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
submission.to_csv('gmb_10fold_selected.csv', index = False )<create_dataframe>
NUM_MODELS = 2
BATCH_SIZE = 512
EPOCHS = 5
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )<filter>
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,y_train,
batch_size=BATCH_SIZE,
epochs=1,
verbose=2 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=BATCH_SIZE))
weights.append(2 ** global_epoch )
print(f"There are {gbm_fi_selected[gbm_fi_selected['importance'] == 0].shape[0]} features with no importance." )<import_modules>
predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )
from hyperopt import hp, tpe, Trials, fmin, STATUS_OK
from hyperopt.pyll.stochastic import sample<import_modules>
sub.iloc[:, 1] =(predictions > 0.5 ).astype(int )
import csv
import ast
from timeit import default_timer as timer<choose_model_class>
sub.to_csv('submission.csv', index=False )
def objective(hyperparameters,nfolds=5):
global ITERATION
ITERATION += 1
subsample = hyperparameters['boosting_type'].get('subsample',1.0)
 subsample_freq = hyperparameters['boosting_type'].get('subsample_freq',0)
 boosting_type = hyperparameters['boosting_type']['boosting_type']
if boosting_type == 'dart':
hyperparameters['drop_rate'] = hyperparameters['boosting_type']['drop_rate']
hyperparameters['subsample'] = subsample
hyperparameters['subsample_freq'] = subsample_freq
hyperparameters['boosting_type'] = boosting_type
if not hyperparameters['limit_max_depth']:
hyperparameters['max_depth'] = -1
for parameter_name in ['max_depth','num_leaves','subsample_for_bin','min_child_samples','subsample_freq']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
strkfold = StratifiedKFold(n_splits=nfolds, shuffle=True)
 features = np.array(train_selected)
 labels = np.array(train_labels ).reshape(( -1))
valid_scores = []
best_estimators = []
run_times =[]
model = lgb.LGBMClassifier(**hyperparameters,class_weight='balanced',n_jobs=-1,metric='None',n_estimators=10000)
 for i,(train_indices,valid_indices)in enumerate(strkfold.split(features,labels)) :
X_train,X_valid = features[train_indices],features[valid_indices]
y_train,y_valid = labels[train_indices],labels[valid_indices]
start = timer()
model.fit(X_train,y_train,early_stopping_rounds=100,eval_metric=macro_f1_score,eval_set=[(X_train,y_train),(X_valid,y_valid)],eval_names=['train','valid'],verbose=400)
 end = timer()
valid_scores.append(model.best_score_['valid']['macro_f1'])
 best_estimators.append(model.best_iteration_)
 run_times.append(end-start)
 score = np.mean(valid_scores)
 score_std = np.std(valid_scores)
 loss = 1 - score
run_time = np.mean(run_times)
 run_time_std = np.std(run_times)
 estimators = int(np.mean(best_estimators))
hyperparameters['n_estimators'] = estimators
of_connection = open(OUT_FILE,'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss,hyperparameters,ITERATION,run_time,score,score_std])
 of_connection.close()
if ITERATION % PROGRESS == 0:
display(f'Iteration: {ITERATION}, Current Score: {round(score, 4)}.')
 return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'time': run_time, 'time_std': run_time_std, 'status': STATUS_OK,
'score': score, 'score_std': score_std}<set_options>
sub.to_csv('submission.csv', index=False )
<find_best_params><EOS> 
Counter(sub['target'] )