Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
Bidirectional, Lambda, Conv1D, MaxPooling1D, GRU,GlobalMaxPooling1D,GlobalAveragePooling1D, concatenate
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
class CyclicLR(Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0<split>
data = pd.read_csv(".. /input/nlp-getting-started/train.csv" )
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()<split>
MAX_SEQUENCE_LENGTH = 60
MAX_NB_WORDS = 30000
EMBEDDING_DIM = 300
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(data['text'].values)
 sequences = tokenizer.texts_to_sequences(data['text'].values)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
pad_text = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH )
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()<split>
embeddings_index = {}
f = open('.. /input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')
 for line in f:
values = line.split(' ')
 word = values[0]
coefs = np.asarray([float(val)for val in values[1:]])
 embeddings_index[word] = coefs
f.close()
print('
Found %s word vectors.' % len(embeddings_index))
embedding_matrix = np.zeros(( len(word_index)+ 1, EMBEDDING_DIM))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Found %s word vectors.' % len(embedding_matrix))
del embedding_vector,embeddings_index
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()<split>
X_train,X_test, y_train, y_test = train_test_split(pad_text,data['target'].values,
test_size=0.33,shuffle=True,random_state=124, stratify=data['target'] )
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()<compute_test_metric>
input_text = Input(shape=(60,),dtype='int64')
 embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix],
trainable=False, mask_zero=True )(input_text)
 text_embed = SpatialDropout1D(0.4 )(embedding_layer)
 hidden_states = Bidirectional(LSTM(units=300, return_sequences=True))(text_embed)
 global_max_pooling = Lambda(lambda x: K.max(x, axis=1))
sentence_embed = global_max_pooling(hidden_states)
 dense_layer = Dense(256, activation='relu' )(sentence_embed)
 output = Dense(1, activation='sigmoid' )(dense_layer)
 BiLSTM = Model(input_text, output)
 BiLSTM.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(6e-6))
lr_callback = CyclicLR()
history = BiLSTM.fit(X_train,y_train, batch_size=5, epochs=3,
validation_data=(X_test,y_test), callbacks=[lr_callback] )
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total / len(train_df)) )<drop_column>
val_preds = BiLSTM.predict(X_test)
 val_preds = np.round(val_preds ).astype(int)
 print(classification_report(y_test,val_preds,target_names = ['Not Relevant', 'Relevant']))
del train_df, weather_train_df, building_meta_df
gc.collect()<feature_engineering>
input_text = Input(shape=(60,),dtype='int64')
 embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix],
trainable=False, mask_zero=True )(input_text)
 text_embed = SpatialDropout1D(0.4 )(embedding_layer)
 conv_layer = Conv1D(300, kernel_size=3, padding="valid", activation='relu' )(text_embed)
 conv_max_pool = MaxPooling1D(pool_size=2 )(conv_layer)
 gru_layer = Bidirectional(GRU(300, return_sequences=True))(conv_max_pool)
 sentence_embed = GlobalMaxPooling1D()(gru_layer)
 dense_layer = Dense(256, activation='relu' )(sentence_embed)
 output = Dense(1, activation='sigmoid' )(dense_layer)
 CNNRNN = Model(input_text, output)
 CNNRNN.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(6e-6))
lr_callback = CyclicLR()
history = CNNRNN.fit(X_train,y_train, batch_size=5, epochs=3,
validation_data=(X_test,y_test), callbacks=[lr_callback] )
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 set_localtime(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_from_disk>
val_preds = CNNRNN.predict(X_test)
 val_preds = np.round(val_preds ).astype(int)
 print(classification_report(y_test,val_preds,target_names = ['Not Relevant', 'Relevant']))
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )<merge>
input_text = Input(shape=(60,),dtype='int64')
 embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix],
trainable=False, mask_zero=True )(input_text)
 text_embed = SpatialDropout1D(0.4 )(embedding_layer)
 gru_layer = Bidirectional(GRU(300, return_sequences=True))(text_embed)
 conv_layer = Conv1D(64, kernel_size=2, padding="valid", kernel_initializer="he_uniform" )(gru_layer)
 avg_pool = GlobalAveragePooling1D()(conv_layer)
 max_pool = GlobalMaxPooling1D()(conv_layer)
 sentence_embed = concatenate([avg_pool, max_pool])
 dense_layer = Dense(256, activation='relu' )(sentence_embed)
 output = Dense(1, activation='sigmoid' )(dense_layer)
 RNNCNN = Model(input_text, output)
 RNNCNN.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=Adam(6e-6))
lr_callback = CyclicLR()
history = RNNCNN.fit(X_train,y_train, batch_size=5, epochs=3,
validation_data=(X_test,y_test), callbacks=[lr_callback] )
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test<predict_on_test>
val_preds = RNNCNN.predict(X_test)
 val_preds = np.round(val_preds ).astype(int)
 print(classification_report(y_test,val_preds,target_names = ['Not Relevant', 'Relevant']))
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total 
<feature_engineering>
train,_, y_train, _ = train_test_split(data['text'].values,data['target'].values,
test_size=0.2,shuffle=True,random_state=124, stratify=data['target'] )
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )<save_to_csv>
MAX_SEQUENCE_LENGTH = 60
MAX_NB_WORDS = 30000
EMBEDDING_DIM = 300
sequences = tokenizer.texts_to_sequences(train)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
pad_text = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH )
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<feature_engineering>
bilstm=BiLSTM.predict(pad_text)
 cr = CNNRNN.predict(pad_text)
 rc = RNNCNN.predict(pad_text )
leak_score0 = 0
leak_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 /= len(leak_df )<feature_engineering>
prediction = pd.DataFrame({"BiLSTM":bilstm.flatten() ,"CR":cr.flatten() ,"RC":rc.flatten() ,"target":y_train} )
leak_score1 = 0
leak_df = pd.read_pickle(ucl_root/'site1.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 /= len(leak_df )<save_to_csv>
clf = Sequential([
Dense(3,activation = 'relu'),
Dense(1,activation= 'sigmoid')
 ])
 clf.compile(loss = 'binary_crossentropy',optimizer = Adam(3e-5),metrics = ['acc'] )
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )<compute_test_metric>
history =clf.fit(prediction.loc[:,['BiLSTM','CR','RC']].values,prediction.iloc[:,-1],validation_split= 0.1,batch_size = 5,epochs = 32 )
print('UCF score = ', np.sqrt(leak_score0))
print('UCL score = ', np.sqrt(leak_score1))<define_variables>
test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
sub_path = ".. /input/ashrae-ensembling-1"
all_files = os.listdir(sub_path)
 all_files<feature_engineering>
MAX_SEQUENCE_LENGTH = 60
MAX_NB_WORDS = 30000
EMBEDDING_DIM = 300
sequences = tokenizer.texts_to_sequences(test.text.values)
 pad_text = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH )
concat_sub['m_max'] = concat_sub.iloc[:, 1:].max(axis=1)
 concat_sub['m_min'] = concat_sub.iloc[:, 1:].min(axis=1)
 concat_sub['m_median'] = concat_sub.iloc[:, 1:].median(axis=1 )<define_variables>
bilstm_test = BiLSTM.predict(pad_text ).flatten()
cr_test = CNNRNN.predict(pad_text ).flatten()
rc_test = RNNCNN.predict(pad_text ).flatten()
cutoff_lo = 0.8
cutoff_hi = 0.2<feature_engineering>
test_predictions = clf.predict(np.stack([bilstm_test,cr_test,rc_test],axis=-1)).flatten()
test_predictions = np.round(test_predictions ).astype(int )
rank = np.tril(concat_sub.iloc[:,1:ncol].corr().values,-1)
 m_gmean = 0
n = 8
while rank.max() >0:
mx = np.unravel_index(rank.argmax() , rank.shape)
 m_gmean += n*(np.log(concat_sub.iloc[:, mx[0]+1])+ np.log(concat_sub.iloc[:, mx[1]+1])) /2
rank[mx] = 0
n += 1<feature_engineering>
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission['target'] = test_predictions
concat_sub['m_mean'] = np.exp(m_gmean/(n-1)**2 )<save_to_csv>
submission.to_csv("/kaggle/working/submission.csv",index = False )