Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
nlp = spacy.load("en")
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
for dirname, _, filenames in os.walk('/kaggle/working'):
for filename in filenames:
print(os.path.join(dirname, filename))
df = pd.DataFrame({'var':np.arange(300),'CV':np.zeros(300),'diff':np.zeros(300)})
 for i in range(300):
df.loc[i,'CV'] = roc_auc_score(train['target'],train[i])
 df.loc[i,'diff'] = abs(df.loc[i,'CV']-0.5)
 print('We need to LB probe',len(df.loc[ df['diff']>0.04 ,'CV']),'variables' )<feature_engineering>
EMBEDDING_SOURCE = "GLOVE"
TUNER_ON = True
DASK_ON = False
df.sort_values('diff',inplace=True,ascending=False)
 LBprobe = list(df.loc[ df['diff']>0.04, 'var'])
 df.sort_values('var',inplace=True)
 df['LB'] = 0.5; df['A'] = 0; ct=0
keep = []
for i in LBprobe:
ct += 1; found = True
df.loc[i,'LB'] = roc_auc_score(public['target'],public[i])
 if(df.loc[i,'LB']<0.47)|(df.loc[i,'LB']>0.53): keep.append(i)
 else: found = False
df.loc[keep,'A'] =(8/9)*df.loc[keep,'LB']+(1/9)*df.loc[keep,'CV']-0.5
predPU = df['A'].values.dot(public.iloc[:,:300].values.transpose())
 aucPU = round(roc_auc_score(public['target'],predPU),3)
 predPR = df['A'].values.dot(private.iloc[:,:300].values.transpose())
 aucPR = round(roc_auc_score(private['target'],predPR),3)
 if found: print('Submission',ct,': Best LB =',aucPU,'and Private score ='
,aucPR,'with',len(keep),'keep' )<feature_engineering>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 train_df.shape
train = pd.read_csv('.. /input/train.csv')
 df = pd.DataFrame({'var':np.arange(300),'CV':np.zeros(300),'diff':np.zeros(300),'LB':0.5*np.ones(300)})
 for i in range(300):
df.loc[i,'CV'] = roc_auc_score(train['target'],train[str(i)])
 df.loc[i,'diff'] = abs(df.loc[i,'CV']-0.5)
 df.sort_values('diff',inplace=True,ascending=False)
 df.head()<feature_engineering>
train_df.keyword.value_counts()
df.loc[ df['var']==33, 'LB' ] = 0.671
df.loc[ df['var']==65, 'LB' ] = 0.671
df.loc[ df['var']==217, 'LB' ] = 0.382
df.loc[ df['var']==117, 'LB' ] = 0.405
df.loc[ df['var']==91, 'LB' ] = 0.382<feature_engineering>
train_df.location.value_counts()
df.loc[ df['var']==295, 'LB' ] = 0.506
df.loc[ df['var']==24, 'LB' ] = 0.501
df.loc[ df['var']==199, 'LB' ] = 0.613
df.loc[ df['var']==80, 'LB' ] = 0.483
df.loc[ df['var']==73, 'LB' ] = 0.394<feature_engineering>
def keyword_cleaner(x):
if type(x)is float:
return 'UNKNOWN'
else:
x = x.replace('%20', ' ')
 doc = nlp(x)
 full_lemma = ' '.join(y.lemma_ for y in doc)
 return full_lemma
train_df['keyword_cleaned'] = train_df['keyword'].map(keyword_cleaner)
 keyword_enc = OneHotEncoder(handle_unknown='ignore')
 transformed = keyword_enc.fit_transform(train_df['keyword_cleaned'].to_numpy().reshape(-1, 1))
ohe_df = pd.DataFrame(transformed.todense() , columns=keyword_enc.get_feature_names())
 meta_features = len(ohe_df.columns)
 ohe_df.to_csv('ohe_train_df.csv', index=False)
 dump(keyword_enc, 'keyword_encoder.joblib')
df.loc[ df['var']==194, 'LB' ] = 0.472
df.loc[ df['var']==189, 'LB' ] = 0.454
df.loc[ df['var']==16, 'LB' ] = 0.437
df.loc[ df['var']==183, 'LB' ] = 0.506
df.loc[ df['var']==82, 'LB' ] = 0.494<feature_engineering>
def location_cleaner(x):
if type(x)is float:
return 'unknown'
else:
x = x.lower().strip()
x = x.replace('?', '')
 if len(x)== 0:
return 'unknown'
x = x.replace('united states', 'usa')
 x = x.replace('usa of america', 'usa')
 x = x.replace('norf carolina', 'north carolina')
 if x == 'tn':
x = 'tennessee'
return x
train_df['location_cleaned'] = train_df['location'].map(location_cleaner)
 location_enc = OneHotEncoder(handle_unknown='ignore')
 transformed = location_enc.fit_transform(train_df['location_cleaned'].to_numpy().reshape(-1, 1))
ohe_location_df = pd.DataFrame(transformed.todense() , columns=location_enc.get_feature_names())
 meta_location_features = len(ohe_location_df.columns)
 ohe_location_df.to_csv('ohe_location_train_df.csv', index=False)
 dump(location_enc, 'location_encoder.joblib' )
df.loc[ df['var']==258, 'LB' ] = 0.474
df.loc[ df['var']==63, 'LB' ] = 0.428
df.loc[ df['var']==298, 'LB' ] = 0.466
df.loc[ df['var']==201, 'LB' ] = 0.489
df.loc[ df['var']==165, 'LB' ] = 0.498<feature_engineering>
train_x = train_df[['text']]
train_y = train_df[['target']]
input_shape = train_x.shape[0]
vocab = set()
for desc in train_df['text'].values:
vocab = vocab.union(set(desc.split(' ')))
 vocab_size = len(vocab)+ 1
print(vocab_size)
 print(train_x.shape)
 word_index_cnt = 1
word_index = {}
seq_of_seqs = []
rows = train_x['text'].tolist()
print(len(rows))
iters = 0
for snippet in train_x['text'].tolist() :
if iters % 1000 == 0:
print(iters)
 doc = nlp(snippet)
 sequences = []
for token in doc:
lower_token = token.text.lower()
if(lower_token not in word_index):
word_index[lower_token] = word_index_cnt
word_index_cnt += 1
sequences.append(word_index[lower_token])
 seq_of_seqs.append(sequences)
 iters += 1
word_index['<OOV>'] = word_index_cnt
word_index_cnt += 1
max_len = 25
train_x_new = pad_sequences(seq_of_seqs, maxlen=max_len)
 print('Done tokenization' )
df.loc[ df['var']==133, 'LB' ] = 0.486
df.loc[ df['var']==209, 'LB' ] = 0.431
df.loc[ df['var']==164, 'LB' ] = 0.542
df.loc[ df['var']==129, 'LB' ] = 0.506
df.loc[ df['var']==134, 'LB' ] = 0.471<feature_engineering>
if EMBEDDING_SOURCE == 'FASTTEXT':
print('loading word embeddings...')
 embeddings_index = {}
with codecs.open('/kaggle/input/fasttext/wiki.simple.vec', encoding='utf-8')as fs:
for line in tqdm(fs):
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print('preparing embedding matrix...')
 MAX_NB_WORDS = 100000
FASTTEXT_EMBEDDING_DIM = 300
words_not_found = []
nb_words = min(MAX_NB_WORDS, len(word_index)+ 1)
 embedding_matrix = np.zeros(( nb_words, FASTTEXT_EMBEDDING_DIM))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if(embedding_vector is not None)and len(embedding_vector)> 0:
embedding_matrix[i] = embedding_vector
else:
words_not_found.append(word)
 print('number of null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1)== 0))
fasttext_embedding_layer = layers.Embedding(nb_words, FASTTEXT_EMBEDDING_DIM, weights=[embedding_matrix], input_length=input_shape, trainable=False )
df.loc[ df['var']==226, 'LB' ] = 0.492
df.loc[ df['var']==237, 'LB' ] = 0.474
df.loc[ df['var']==39, 'LB' ] = 0.488
df.loc[ df['var']==17, 'LB' ] = 0.508
df.loc[ df['var']==30, 'LB' ] = 0.494<feature_engineering>
if EMBEDDING_SOURCE == 'GLOVE':
print('Creating Glove Embedding Layer')
 embeddings_index = {}
GLOVE_DIR = '/kaggle/input/glove-global-vectors-for-word-representation'
GLOVE_EMBEDDING_DIM = 200
with open(os.path.join(GLOVE_DIR, 'glove.6B.200d.txt')) as fs:
lines = fs.readlines()
for line in lines:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
hits = 0
misses = 0
missed_words = []
embedding_matrix = np.zeros(( len(word_index)+ 1, GLOVE_EMBEDDING_DIM))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
hits += 1
else:
misses += 1
missed_words.append(word)
 print("Converted %d words(%d misses)" %(hits, misses))
print(missed_words[0:100])
 glove_embedding_layer = layers.Embedding(
 len(word_index)+ 1,
GLOVE_EMBEDDING_DIM,
embeddings_initializer=initializers.Constant(embedding_matrix),
input_length=input_shape,
trainable=False,
)
df.loc[ df['var']==114, 'LB' ] = 0.479
df.loc[ df['var']==272, 'LB' ] = 0.471
df.loc[ df['var']==108, 'LB' ] = 0.459
df.loc[ df['var']==220, 'LB' ] = 0.511
df.loc[ df['var']==150, 'LB' ] = 0.482<feature_engineering>
if EMBEDDING_SOURCE == 'BERT':
print('Creating BERT Embedding Layer')
 BERT_EMBEDDING_DIM = 256
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
 iters = 0
ii_list = []
am_list = []
tti_list = []
for snippet in train_x['text'].tolist() :
if iters % 1000 == 0:
print(iters)
 encoding = tokenizer(snippet)
 input_ids = encoding['input_ids']
token_type_ids = [0] * len(encoding['input_ids'])
 attention_mask = [1] * len(input_ids)
 padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 ii_list.append(input_ids)
 am_list.append(attention_mask)
 tti_list.append(token_type_ids)
 iters += 1
encoder = TFBertModel.from_pretrained("bert-base-uncased")
 print(encoder)
 input_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 token_type_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 attention_mask = layers.Input(shape=(max_len,), dtype=tf.int32)
 sequence_output, pooled_output = encoder({
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": attention_mask
})
 print(type(seq_output))
print(type(pooled_output))
print(type(encoder.bert))
bert_embedding_layer = encoder.bert
print('Done creating BERT Embedding layer' )
df.loc[ df['var']==230, 'LB' ] = 0.482
df.loc[ df['var']==90, 'LB' ] = 0.486
df.loc[ df['var']==289, 'LB' ] = 0.482
df.loc[ df['var']==241, 'LB' ] = 0.494
df.loc[ df['var']==4, 'LB' ] = 0.516<feature_engineering>
if EMBEDDING_SOURCE == 'CUSTOM':
CUSTOM_EMBEDDING_DIM = 512
training_data_embedding_layer = layers.Embedding(input_dim = vocab_size, output_dim = CUSTOM_EMBEDDING_DIM, input_length = input_shape )
df.loc[ df['var']==43, 'LB' ] = 0.474
df.loc[ df['var']==239, 'LB' ] = 0.457
df.loc[ df['var']==127, 'LB' ] = 0.501
df.loc[ df['var']==45, 'LB' ] = 0.449
df.loc[ df['var']==151, 'LB' ] = 0.504<feature_engineering>
def build_model(hp):
nlp_input = layers.Input(shape=(input_shape,), name='nlp_input')
 meta_input = layers.Input(shape=(meta_features + meta_location_features,), name='meta_input')
 if EMBEDDING_SOURCE == 'GLOVE':
print('GLOVE EMBARKED')
 FINAL_EMBEDDING_DIM = GLOVE_EMBEDDING_DIM
embedding_layer = glove_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'BERT':
print('BERT EMBARKED')
 FINAL_EMBEDDING_DIM = BERT_EMBEDDING_DIM
embedding_layer = bert_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'FASTTEXT':
print('FASTTEXT EMBARKED')
 FINAL_EMBEDDING_DIM = FASTTEXT_EMBEDDING_DIM
embedding_layer = fasttext_embedding_layer(nlp_input)
 else:
FINAL_EMBEDDING_DIM = CUSTOM_EMBEDDING_DIM
embedding_layer = training_data_embedding_layer(nlp_input)
 lstm_layer = layers.LSTM(FINAL_EMBEDDING_DIM, dropout=0.1 )(embedding_layer)
 merged = layers.Concatenate(axis=1 )([lstm_layer, meta_input])
 hp_dense_units = hp.Int('units', min_value = 32, max_value = 512, step = 32)
 dense_layer = layers.Dense(hp_dense_units, activation='relu' )(merged)
 output_layer = layers.Dense(1, activation='sigmoid' )(dense_layer)
 model = Model(inputs=[nlp_input, meta_input], outputs=[output_layer])
 hp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-3, 1e-4])
 model.compile(loss='binary_crossentropy', optimizer = Adam(learning_rate=hp_learning_rate), metrics=['accuracy'],)
 return model
df.loc[ df['var']==244, 'LB' ] = 0.509
df.loc[ df['var']==26, 'LB' ] = 0.511
df.loc[ df['var']==105, 'LB' ] = 0.505
df.loc[ df['var']==176, 'LB' ] = 0.525
df.loc[ df['var']==101, 'LB' ] = 0.535<feature_engineering>
def build_model_wrapper(optimizer="adam", embedding_size = 512, dense_size=32, dropout_value = 0.1):
nlp_input = layers.Input(shape=(input_shape,), name='nlp_input')
 meta_input = layers.Input(shape=(meta_features + meta_location_features,), name='meta_input')
 if EMBEDDING_SOURCE == 'GLOVE':
print('GLOVE EMBARKED')
 FINAL_EMBEDDING_DIM = GLOVE_EMBEDDING_DIM
embedding_layer = glove_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'BERT':
print('BERT EMBARKED')
 FINAL_EMBEDDING_DIM = BERT_EMBEDDING_DIM
embedding_layer = bert_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'FASTTEXT':
print('FASTTEXT EMBARKED')
 FINAL_EMBEDDING_DIM = FASTTEXT_EMBEDDING_DIM
embedding_layer = fasttext_embedding_layer(nlp_input)
 else:
FINAL_EMBEDDING_DIM = CUSTOM_EMBEDDING_DIM
embedding_layer = training_data_embedding_layer(nlp_input)
 lstm_layer = layers.LSTM(FINAL_EMBEDDING_DIM, dropout=dropout_value )(embedding_layer)
 merged = layers.Concatenate(axis=1 )([lstm_layer, meta_input])
 dense_layer = layers.Dense(dense_size, activation='relu' )(merged)
 output_layer = layers.Dense(1, activation='sigmoid' )(dense_layer)
 model = Model(inputs=[nlp_input, meta_input], outputs=output_layer)
 model.compile(loss='binary_crossentropy', optimizer = optimizer, metrics=['accuracy'],)
 print(model.summary())
 return model
df['A'] = 0
df['A'] =(8/9)*df['LB'] +(1/9)*df['CV'] - 0.500
keep_threshold = 0.04
df.loc[ abs(df['A'])<keep_threshold , 'A' ] = 0
df.sort_values('var',inplace=True)
 for i in range(300):
if df.loc[i,'LB'] != 0.500:
print('A_'+str(i)+' = ',round(df.loc[i,'A'],6))<save_to_csv>
if TUNER_ON is True:
tuner = RandomSearch(build_model,
objective='val_loss',
max_trials=18,
executions_per_trial=1,
)
test = pd.read_csv('.. /input/test.csv')
 pred = test.iloc[:,1:].values.dot(df['A'].values)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = pred
sub.to_csv('submission.csv',index=False )<load_from_csv>
tf.config.run_functions_eagerly(True)
 epochs = 5
batch_size = 64
es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
 X_train_pure, X_valid, y_train_pure, y_valid = train_test_split(train_x_new, train_y, test_size = 0.15, random_state=0)
 ohe_train, ohe_valid = train_test_split(ohe_df, test_size = 0.15, random_state=0)
 ohe_location_train, ohe_location_valid = train_test_split(ohe_location_df, test_size = 0.15, random_state=0)
 print(ohe_train.shape)
 print(ohe_location_train.shape)
 meta_train = pd.concat([ohe_train, ohe_location_train], axis=1, sort=False)
 meta_valid = pd.concat([ohe_valid, ohe_location_valid], axis=1, sort=False)
 if TUNER_ON is True:
tuner.search([X_train_pure, meta_train], y_train_pure, epochs=epochs, callbacks=[es_callback], validation_data =([X_valid, meta_valid], y_valid))
best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]
model_real = tuner.hypermodel.build(best_hps)
 else:
print('Using KerasClassifier method to select best hyperparameters for model')
 clf = KerasClassifier(build_fn=build_model_wrapper)
 parameters = {
'epochs': [10],
'batch_size':[32],
'dense_size':[32],
'optimizer':['Adam'],
'dropout_value': [0.1]
}
kfold_splits = 2
grid_search = GridSearchCV(estimator=clf,
n_jobs=-1,
verbose=1,
return_train_score=True,
cv=kfold_splits,
param_grid=parameters,)
 combi_input = np.concatenate([X_train_pure, meta_train], axis=1)
 if DASK_ON is True:
with joblib.parallel_backend('dask'):
grid_search.fit(combi_input, y_train_pure)
 else:
grid_search.fit(combi_input, y_train_pure)
 model_real = grid_search.best_estimator_
print(grid_search.best_parameters_)
 model_real.summary()
train_df = pd.read_csv('.. /input/train.csv', index_col='id')
 test_df = pd.read_csv('.. /input/test.csv', index_col='id' )<train_model>
if DASK_ON is True:
with joblib.parallel_backend('dask'):
fit_history = model_real.fit([X_train_pure, meta_train], y_train_pure, batch_size = batch_size, epochs=epochs, callbacks=[es_callback], validation_data =([X_valid, meta_valid], y_valid), verbose=1)
 else:
fit_history = model_real.fit([X_train_pure, meta_train], y_train_pure, batch_size = batch_size, epochs=epochs, callbacks=[es_callback], validation_data =([X_valid, meta_valid], y_valid), verbose=1 )
print("number of training samples: ", train_df.shape[0])
 print("number of testing samples: ", test_df.shape[0] )<count_values>
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test_df.shape
print("count of target values: ")
 train_df.target.value_counts()<define_variables>
test_df['keyword_cleaned'] = test_df['keyword'].map(keyword_cleaner)
 transformed = keyword_enc.transform(test_df['keyword_cleaned'].to_numpy().reshape(-1, 1))
ohe_df = pd.DataFrame(transformed.todense() , columns=keyword_enc.get_feature_names())
 ohe_df.to_csv('ohe_test_df.csv', index=False )
def bootstrap(data, target, n=5):

 samples = []
for i in range(n):
ix = range(len(data))
random_ixs = np.random.choice(ix, len(data), replace=True)
 new_data, new_target = data[random_ixs, :], target[random_ixs]
samples.append(( new_data, new_target))
return samples<define_variables>
test_df['location_cleaned'] = test_df['location'].map(location_cleaner)
 transformed = location_enc.transform(test_df['location_cleaned'].to_numpy().reshape(-1, 1))
ohe_location_df = pd.DataFrame(transformed.todense() , columns=location_enc.get_feature_names())
 ohe_location_df.to_csv('ohe_location_test_df.csv', index=False )
features = [col for col in train_df.columns if col.isdigit() ]
target = 'target'<compute_test_metric>
X_test = test_df[['text']]
test_sequences = []
for snippet in X_test['text'].tolist() :
sequences = []
doc = nlp(snippet)
 for token in doc:
lower_token = token.text.lower()
if(lower_token not in word_index):
sequences.append(word_index['<OOV>'])
 else:
sequences.append(word_index[lower_token])
 test_sequences.append(sequences)
 X_test_new = pad_sequences(test_sequences, maxlen=max_len)
 meta_test = pd.concat([ohe_df, ohe_location_df], axis=1, sort=False)
 preds = model_real.predict([X_test_new, meta_test])
 preds = np.ndarray.round(preds ).astype(int)
 preds
max_alpha_ix = np.argmax(np.vstack(( mean_train_auc_roc, mean_val_auc_roc)).mean(axis=0))
max_mean_cv_score = np.max(np.vstack(( mean_train_auc_roc, mean_val_auc_roc)).mean(axis=0))
alpha = alpha_values[max_alpha_ix]
print("alpha: ", alpha, "max_mean, cv_score: ", max_mean_cv_score )<compute_test_metric>
sub_df = pd.DataFrame({'id': test_df.id, 'target': preds.flatten() })
 sub_df.to_csv('submission21.csv', index=False )