Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = predictions[feats].mean(axis=1)
 y['mymedian'] = predictions[feats].median(axis=1)
 y['mymax'] = predictions[feats].max(axis=1)
 predictions['class_99'] = GenUnknown(y )<merge>
tweet = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 tweet.head()
def modUnknown(opdf, meta, ddfMult=0.5, mwMult=0.5, preserveMed=False):
pdf=copy.deepcopy(opdf)
 mdf=pdf.merge(meta,on='object_id')
 ddfilter=mdf.loc[:,'ddf']==1
mwfilter=mdf.loc[:,'hostgal_photoz']==0
print(ddfilter.sum())
 print(mwfilter.sum())
 mdf.loc[mwfilter,'class_99']=mwMult*mdf.loc[mwfilter,'class_99']
mdf.loc[ddfilter,'class_99']=ddfMult*mdf.loc[ddfilter,'class_99']
pdf.loc[:,'class_99']=mdf.loc[:,'class_99']
return pdf
npdf=modUnknown(predictions, temeta)
 npdf.head()<save_to_csv>
df = pd.concat([tweet,test])
 df.shape
npdf.to_csv('probaZeroAndRe99.csv', index=False )<load_from_csv>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table )
dfparam = pd.read_csv(".. /input/ourpathtowherewearenewparams/subm_0.610175_2018-12-07-23-29.csv")
 dforig = pd.read_csv(".. /input/ourpathtowhereweare/subm_0.619987_2018-12-06-19-58.csv")
 dfsix = pd.read_csv(".. /input/sixdeepourpathtowherewearenewparams/subm_0.613687_2018-12-07-04-17.csv")
 dffour = pd.read_csv(".. /input/depthfourourpathtowherewearenewparams/subm_0.615926_2018-12-07-16-59.csv")
 dfxgb= pd.read_csv(".. /input/xgb-from-ideas-from-kernels-and-discussion/single_subm_0.671072_XGB_2018-11-21-14-24.csv")
 dfsm = pd.read_csv(".. /input/smote-on-lgbm-w-ideas-from-kernels-and-discussion/single_subm_0.751366_2018-12-10-14-48.csv")
 dfnn = pd.read_csv(".. /input/trypredictingallatoncenn71/subm_0.639462_2018-12-10-23-23.csv")
<save_to_csv>
df['text'] = df['text'].apply(lambda x : remove_URL(x))
df['text'] = df['text'].apply(lambda x : remove_html(x))
df['text'] = df['text'].apply(lambda x : remove_emoji(x))
df['text'] = df['text'].apply(lambda x : remove_punct(x))
df_blend =.3*dfparam +.075*(dforig+dfsix+dffour)+.2 *(dfxgb + dfnn)
 df_blend['object_id']=dfsm['object_id']
df_blend.to_csv('blend_submission.csv',index=False )<import_modules>
train = df[:tweet.shape[0]]
test = df[tweet.shape[0]:]
np.warnings.filterwarnings('ignore')
<set_options>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
np.warnings.filterwarnings('ignore')
<categorify>
import tokenization
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss<compute_test_metric>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
def lgbm_multi_weighted_logloss(y_true, y_preds):

 classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted,
classes, class_weights)
 return 'wloss', loss<compute_train_metric>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def xgb_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=7,
random_state=1):
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
func_loss = partial(xgb_multi_weighted_logloss,
classes=classes,
class_weights=class_weights)
 clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
clf = XGBClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=func_loss,
verbose=100,
early_stopping_rounds=77,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, ntree_limit=clf.best_ntree_limit)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('xgb_importances.csv', index=False)
 return clfs, score<train_model>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):
sm=SMOTE(random_state=2)
 Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())
 return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test )<compute_train_metric>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def lgbm_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=7,
random_state=1):
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
trn_xa, trn_y, val_xa, val_y=smoteAdataset(trn_x.values, trn_y.values, val_x.values, val_y.values)
 trn_x=pd.DataFrame(data=trn_xa, columns=trn_x.columns)
 val_x=pd.DataFrame(data=val_xa, columns=val_x.columns)
 clf = LGBMClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgbm_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=77,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('lgbm_importances.csv', index=False)
 return clfs, score, oof_preds, importances<predict_on_test>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
test_input = bert_encode(test.text.values, tokenizer, max_len=160 )
def predict_chunk(df_, clfs_, features, train_mean):
agg_ = df_
full_test = agg_.reset_index()
print(full_test.head())
 full_test = full_test.fillna(0)
 preds_ = None
for clf in clfs_:
if preds_ is None:
preds_ = clf.predict_proba(full_test[features])/ len(clfs_)
 else:
preds_ += clf.predict_proba(full_test[features])/ len(clfs_)
 preds_99 = np.ones(preds_.shape[0])
 for i in range(preds_.shape[1]):
preds_99 *=(1 - preds_[:, i])
 preds_df_ = pd.DataFrame(preds_, columns=['class_' + str(s)for s in clfs_[0].classes_])
 preds_df_['object_id'] = full_test['object_id']
preds_df_['class_99'] = 0.14 * preds_99 / np.mean(preds_99)
 return preds_df_<save_to_csv>
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16 
)
 model.save('model.h5' )
def process_test(clfs,
testdf,
full_train,
train_mean,
filename='submission.csv',
chunks=40615):
start = time.time()
chunks = 40615
testdf=testdf.round(4)
 testdf.to_csv(filename, index=False)
 for i_c, df in enumerate(pd.read_csv(filename, chunksize=chunks, iterator=True)) :
print(df.shape)
 preds_df = predict_chunk(df_=df,
clfs_=clfs,
features=full_train.columns,
train_mean=train_mean)
 if i_c == 0:
preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)
 else:
preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)
 del preds_df
gc.collect()
print('%15d done in %5.1f minutes' %(chunks *(i_c + 1),(time.time() - start)/ 60), flush=True)
 return<categorify>
test_pred = model.predict(test_input )
<init_hyperparams><EOS> 
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )