Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
for dirname, _, filenames in os.walk('/kaggle/working'):
for filename in filenames:
print(os.path.join(dirname, filename))
print('นำเข้าไลบรารี่ข้อมูลเรียบร้อย' )
types_dict = {
"1JHC": 0,
"2JHH": 3,
"1JHN": 1,
"2JHN": 4,
"2JHC": 2,
"3JHH": 6,
"3JHC": 5,
"3JHN": 7,
}
atom_features = [
"atom_2",
"atom_3",
"atom_4",
"atom_5",
"atom_6",
"atom_7",
"atom_8",
"atom_9",
]
fc_feats = [
"fc_preds_type",
"fc_preds_akira",
"fc_preds_akira2",
"fc_preds_akira3",
"fc_preds_akira4",
]
types_to_run = ["1JHN", "1JHC", "2JHH", "2JHN", "2JHC", "3JHH", "3JHC", "3JHN"]
all_feats = []
for mol_type in types_to_run:
i = types_dict[mol_type]
linear_feats = select_features_for_type(
 i, top_n=LINEAR_TOP, imp_threshold=0,
imp_file=".. /input/cached-champs/feats_inv_pvals.csv" 
)
 lgb_feats = select_features_for_type(
 i, top_n=LGB_TOP, imp_threshold=0, 
)
 feats = linear_feats + lgb_feats
all_feats.append(feats)
 all_feats = np.unique(list(np.concatenate(all_feats)) + atom_features + fc_feats)
 print(all_feats )<load_from_csv>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sample_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print('นำเข้าชุดข้อมูลเรียบร้อย' )
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 cols = ['molecule_name', 'atom_index_0', 'atom_index_1', 'type']
train_csv = train_csv[cols]
train_csv.head(10)
<load_from_csv>
print(train.isnull().sum() )
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 cols = [col for col in cols if 'scalar_coupling_constant' not in col]
test_csv = test_csv[cols]
gc.collect()
print(train_csv.shape)
 print(test_csv.shape)
 test_csv.head()<feature_engineering>
print(test.isnull().sum() )
train_csv, test_csv = add_contributions(train_csv, test_csv )<drop_column>
print(sample_submission.isnull().sum() )
cols = ["molecule_name", "atom_index_0", "atom_index_1"]
train_csv = train_csv.drop(cols, axis=1)
 test_csv = test_csv.drop(cols, axis=1)
 train_csv.head()<load_from_csv>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
print('ดาวน์โหลด Algorithm สำเร็จ!!!' )
tr = read_pickle(FILETRAIN)
 tr = tr.fillna(0)
 train_ix = train_csv.index
tr.index = train_ix
train_csv = pd.concat([train_csv, tr], axis=1)
 train_csv.index = train_ix
train_csv = train_csv[[col for col in train_csv.columns if col in list(all_feats)+ TARGETS + ["type"]]]
del tr<load_from_csv>
print('นำเข้า "tokenization" เรียบร้อย' )
te = read_pickle(FILETEST)
 te = te.fillna(0)
 test_ix = test_csv.index
te.index = test_ix
test_csv = pd.concat([test_csv, te], axis=1)
 test_csv.index = test_ix
test_csv = test_csv[[col for col in test_csv.columns if col in list(all_feats)+ ["type"]]]
del te, test_ix, train_ix
gc.collect()
print(train_csv.shape)
 print(test_csv.shape)
 test_csv.head(10 )<load_from_csv>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]

 input_sequence = ["[CLS]"] + text + ["[SEP]"]

 pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 print("เสร็จสิ้น" )
tr = read_pickle(FILETRAIN1)
 tr = tr.fillna(0)[[col for col in tr.columns if col not in train_csv.columns]]
train_ix = train_csv.index
tr.index = train_ix
train_csv = pd.concat([train_csv, tr], axis=1)
 train_csv.index = train_ix
del tr
train_csv = train_csv[list(all_feats)+ TARGETS + ["type"]]
gc.collect()
te = read_pickle(FILETEST1)
 te = te.fillna(0)[[col for col in te.columns if col not in test_csv.columns]]
test_ix = test_csv.index
te.index = test_ix
test_csv = pd.concat([test_csv, te], axis=1)
 test_csv.index = test_ix
del te, test_ix, train_ix
gc.collect()
test_csv = test_csv[list(all_feats)+ ["type"]]
test_csv.head(10)
 print(train_csv.shape)
 print(test_csv.shape)
 gc.collect()<count_unique_values>
"Let's learn deep learning!"
['Let', "'", 's', 'learn', 'deep', 'learning', '!']
['[CLS]', 'Let', "'", 's', 'learn', 'deep', 'learning', '!', '[SEP]']
['[CLS]', 'Let', "'", 's', 'learn', 'deep', 'learning', '!', '[SEP]', '[PAD]','[PAD]','[PAD]','[PAD]','[PAD]']
[101, 2421, 112, 188, 3858, 1996, 3776, 106, 102, 0, 0, 0, 0, 0 ,0 ]
gc.collect()
print(len(train_csv.columns), len(np.unique(train_csv.columns)) )<set_options>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
print("เสร็จสิ้น" )
config = tf.ConfigProto(device_count = {'GPU': 1 , 'CPU': 2})
 config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.6
sess = tf.Session(config=config)
 K.set_session(sess )<categorify>
%%time
print('กำลังดาวน์โหลดโมเดลอาจใช้เวลาสักครู่...')
 module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 print('ดาวน์โหลดโมเดลสำเร็จ!!!' )
cv_score = []
cv_score_total = 0
retrain = True
start_time = datetime.now()
test_prediction = np.zeros(len(test_csv))
class FeatureTransformer:
def transform(self, dataset, ohe_features=[], continuous_features=[]):
ohe_df = OneHotEncoder().fit_transform(dataset.loc[:, ohe_features] ).toarray()
skews = dataset.loc[:, continuous_features].skew().to_dict()
return np.concatenate(
 [
StandardScaler().fit_transform(dataset.loc[:, continuous_features]),
ohe_df,
],
axis=1, 
)
 for mol_type in types_to_run:
model_name_wrt = "/kaggle/working/molecule_model_%s.hdf5" % mol_type
model_name_rd = ".. /input/nncont-seed-25/molecule_model_%s.hdf5" % mol_type
if mol_type=="3JHC":
linear = 16
lgb = 16
else:
linear = LINEAR_TOP
lgb = LGB_TOP
linear_feats = select_features_for_type(
 types_dict[mol_type], top_n=linear, imp_threshold=0,
imp_file=".. /input/cached-champs/feats_inv_pvals.csv" 
)
 lgb_feats = select_features_for_type(
 types_dict[mol_type], top_n=lgb, imp_threshold=0, 
)
 distance_features = list(np.unique(linear_feats + lgb_feats + fc_feats))
input_features = list(np.unique(distance_features + atom_features))
print(input_features)
 df_train_ = train_csv.loc[train_csv["type"] == mol_type, input_features + TARGETS]
df_test_ = test_csv.loc[test_csv["type"] == mol_type, input_features]
msg = f"training {mol_type}, out of {types_to_run}, train shape: {df_train_.shape}, LR: {LR}"
print(msg)
 logger.debug(msg)
 input_data = FeatureTransformer().transform(
 dataset=pd.concat(
 [df_train_.loc[:, input_features], df_test_.loc[:, input_features]] 
),
ohe_features=atom_features,
continuous_features=distance_features, 
)
 target_data = df_train_.loc[:, TARGETS].values
train_index, cv_index = train_test_split(
 np.arange(len(df_train_)) , random_state=SEED, test_size=0.1 
)
 train_target = target_data[train_index]
cv_target = target_data[cv_index]
train_input = input_data[train_index]
cv_input = input_data[cv_index]
test_input = input_data[len(df_train_):, :]
nn_model = create_nn_model(train_input.shape[1])
 retrain = False
if not retrain:
nn_model = load_model(model_name_rd)
 nn_model.compile(loss="mae", optimizer=Adam(lr=LR))
es = callbacks.EarlyStopping(
 monitor="val_loss",
min_delta=0.0001,
patience=50,
verbose=1,
mode="auto",
restore_best_weights=True, 
)
 rlr = callbacks.ReduceLROnPlateau(
 monitor="val_loss", factor=0.5, patience=30, min_lr=1e-7, mode="auto", verbose=1 
)
 sv_mod = callbacks.ModelCheckpoint(
 model_name_wrt, monitor="val_loss", save_best_only=True, period=1 
)
 history = nn_model.fit(
 train_input,
[train_target],
validation_data=(cv_input, [cv_target]),
callbacks=[es, rlr, sv_mod],
epochs=epoch_n,
batch_size=batch_size,
verbose=0, 
)
 cv_predict = nn_model.predict(cv_input ).sum(axis=1)
 plot_history(history, mol_type)
 accuracy = np.mean(np.abs(cv_target.sum(axis=1)- cv_predict))
print(np.log(accuracy))
cv_score.append(np.log(accuracy))
cv_score_total += np.log(accuracy)
 test_predict = nn_model.predict(test_input ).sum(axis=1)
 test_prediction[test_csv["type"] == mol_type] = test_predict
K.clear_session()
cv_score_total /= len(types_to_run )<save_to_csv>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')
 def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv("/kaggle/working/nnetCont_sub.csv", index=False)
 submits(test_prediction )<train_model>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
print('Total training time: ', datetime.now() - start_time)
 i=0
for mol_type in types_to_run:
print(mol_type,": cv score is ",cv_score[i])
 i+=1
print("total cv score is",cv_score_total )<save_to_csv>
start_time = time.time()
train_history = model.fit(train_input,
train_labels,
validation_split = 0.2,
epochs = 3,
batch_size = 16)
 print(' สำเร็จ!!! ')
 print("---ใช้เวลาทั้งหมด %s วินาที ---" %(time.time() - start_time))
submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')
 def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv(f"/kaggle/working/nnetCont_sub_{round(cv_score_total, 4)}.csv", index=False)
 submits(test_prediction )<set_options>
test_pred = model.predict(test_input)
 print(' สำเร็จ!!! ' )
%%capture
warnings.filterwarnings('ignore')
 DATA_DIR = '.. /input/champs-scalar-coupling'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<load_from_csv>
sample_submission['target'] = test_pred.round().astype(int)
 sample_submission.to_csv('sample_submission.csv', index=False)
 print('สร้างไฟล์ submission.csv เรียบร้อย' )
<merge><EOS> 
sample_submission.isnull().sum()