Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
X_train = prepare_data(df_train, building, weather_train )<set_options>
SEED = 1002
def seed_everything(seed):
np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )
del df_train
gc.collect()<create_dataframe>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 for i, val in enumerate(train.iloc[:2]["text"].to_list()):
print("Tweet {}: {}".format(i+1, val))
def LGBM(X_train, y_train):
X_half_1 = X_train[:int(X_train.shape[0] / 2)]
X_half_2 = X_train[int(X_train.shape[0] / 2):]
y_half_1 = y_train[:int(X_train.shape[0] / 2)]
y_half_2 = y_train[int(X_train.shape[0] / 2):]
categorical_features = ["hour", "weekday"]
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)
 watchlist_1 = [d_half_1, d_half_2]
watchlist_2 = [d_half_2, d_half_1]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 40,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse"
}
model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200)
 return model_half_1, model_half_2<create_dataframe>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
def modeling(X_train, stop=999999):
drop_list = ["building_id", "meter", "meter_reading"]
ids = np.sort(X_train["building_id"].unique())
 df_model = pd.DataFrame(columns=["building_id", "meter", "half_1", "half_2"])
 index = 0
for bid in ids:
if stop < bid:
break
meters = np.sort(X_train[X_train["building_id"] == bid]["meter"].unique())
 for meter in meters:
Modeling building_id:{}, meter:{}
---------
".format(bid, meter))
X_each = X_train[(X_train["building_id"] == bid)&(X_train["meter"] == meter)]
y_each = np.log1p(X_each.meter_reading)
 X_each.drop(drop_list, axis=1, inplace=True)
 model_half_1, model_half_2 = LGBM(X_each, y_each)
 df_model.loc[index] = [bid, meter, model_half_1, model_half_2]
index+=1
return df_model<load_pretrained>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(learning_rate=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
df_model = pd.read_pickle(path_model)
 df_model<features_selection>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
drop_list = ["building_id", "meter", "meter_reading"]
model_names = ["half_1", "half_2"]
colmuns = X_train[(X_train["building_id"] == 0)&(X_train["meter"] == 0)].drop(drop_list, axis=1 ).columns.values
for bid in [31, 15, 75]:
meters = np.sort(X_train[X_train["building_id"] == bid]["meter"].unique())
 for meter in meters:
models = []
for m_name in model_names:
model = df_model[(df_model["building_id"] == bid)&(df_model["meter"] == meter)][m_name].values[0]
models.append(model)
 feature_importance(colmuns, models, bid, meter )<set_options>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
del X_train
gc.collect()<load_from_csv>
checkpoint = ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.1,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)
<set_options><EOS> 
test_pred = model.predict(test_input)
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )