Kaggle competition: Natural Language Processing with Disaster Tweets
bureau['LOAN_RATE'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_SUM']<prepare_output>
!pip install -q tf-nightly
!pip install -q tf-models-nightly
bureau_info = agg_child(bureau, 'SK_ID_CURR', 'BUREAU')
 bureau_info.head()<feature_engineering>
tfds.disable_progress_bar()
bureau_balance['PAST_DUE'] = bureau_balance['STATUS'].isin(['1', '2', '3', '4', '5'])
 bureau_balance['ON_TIME'] = bureau_balance['STATUS'] == '0'<merge>
train = ".. /input/nlp-getting-started/train.csv"
test = ".. /input/nlp-getting-started/test.csv"
bureau_balance_info = agg_grandchild(bureau_balance, bureau, 'SK_ID_BUREAU', 'SK_ID_CURR', 'BB')
 del bureau_balance, bureau
bureau_balance_info.head()<merge>
data =(pd.read_csv(train)).iloc[:,3:]
data.shape
app = app.set_index('SK_ID_CURR')
 app = app.merge(bureau_info, on = 'SK_ID_CURR', how = 'left')
 del bureau_info
app.shape<merge>
tokenizer = bert.tokenization.FullTokenizer(
 vocab_file=os.path.join(gs_folder_bert, "vocab.txt"),
do_lower_case=True )
app = app.merge(bureau_balance_info, on = 'SK_ID_CURR', how = 'left')
 del bureau_balance_info
app.shape<load_from_csv>
def encode_sentence(s):
tokens = list(tokenizer.tokenize(s))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens)
 sentence = tf.ragged.constant([
encode_sentence(s)for s in data['text']] )
previous = pd.read_csv('.. /input/previous_application.csv' ).replace({365243: np.nan})
 previous = convert_types(previous)
 previous['LOAN_RATE'] = previous['AMT_ANNUITY'] / previous['AMT_CREDIT']
previous["AMT_DIFFERENCE"] = previous['AMT_CREDIT'] - previous['AMT_APPLICATION']<merge>
cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*sentence.shape[0]
input_word_ids = tf.concat([cls, sentence], axis=-1)
 _ = plt.pcolormesh(input_word_ids.to_tensor())
app = app.merge(previous_info, on = 'SK_ID_CURR', how = 'left')
 del previous_info
app.shape<load_from_csv>
input_mask = tf.ones_like(input_word_ids ).to_tensor()
plt.pcolormesh(input_mask )
installments = pd.read_csv('.. /input/installments_payments.csv' ).replace({365243: np.nan})
 installments = convert_types(installments)
 installments['LATE'] = installments['DAYS_ENTRY_PAYMENT'] > installments['DAYS_INSTALMENT']
installments['LOW_PAYMENT'] = installments['AMT_PAYMENT'] < installments['AMT_INSTALMENT']<merge>
def encode_sentence(s, tokenizer):
tokens = list(tokenizer.tokenize(s))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens)
 def bert_encode(data, tokenizer):
num_examples = len(data["text"])
 sentence = tf.ragged.constant([
encode_sentence(s, tokenizer)
 for s in np.array(data["text"])])
 cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*sentence.shape[0]
input_word_ids = tf.concat([cls, sentence], axis=-1)
 input_mask = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.zeros_like(cls)
 type_s = tf.zeros_like(sentence)
 input_type_ids = tf.concat(
 [type_cls, type_s], axis=-1 ).to_tensor()
inputs = {
'input_word_ids': input_word_ids.to_tensor() ,
'input_mask': input_mask,
'input_type_ids': input_type_ids}
return inputs
app = app.merge(installments_info, on = 'SK_ID_CURR', how = 'left')
 del installments_info
app.shape<load_from_csv>
d_train,d_val,d_test = np.split(data.sample(frac=1), [int (.6*len(data)) , int (.8*len(data)) ] )
cash = pd.read_csv('.. /input/POS_CASH_balance.csv' ).replace({365243: np.nan})
 cash = convert_types(cash)
 cash['LATE_PAYMENT'] = cash['SK_DPD'] > 0.0
cash['INSTALLMENTS_PAID'] = cash['CNT_INSTALMENT'] - cash['CNT_INSTALMENT_FUTURE']<concatenate>
data_train = bert_encode(d_train, tokenizer)
 data_train_labels = d_train['target']
data_validation = bert_encode(d_val, tokenizer)
 data_validation_labels = d_val['target']
data_test = bert_encode(d_test, tokenizer)
 data_test_labels = d_test['target']
cash_info = agg_grandchild(cash, previous, 'SK_ID_PREV', 'SK_ID_CURR', 'CASH')
 del cash
cash_info.shape<merge>
config_dict = {'attention_probs_dropout_prob': 0.1,
'hidden_act': 'gelu',
'hidden_dropout_prob': 0.1,
'hidden_size': 768,
'initializer_range': 0.02,
'intermediate_size': 3072,
'max_position_embeddings': 512,
'num_attention_heads': 12,
'num_hidden_layers': 12,
'type_vocab_size': 2,
'vocab_size': 30522}
bert_config = bert.configs.BertConfig.from_dict(config_dict)
 config_dict
app = app.merge(cash_info, on = 'SK_ID_CURR', how = 'left')
 del cash_info
app.shape<feature_engineering>
bert_classifier, bert_encoder = bert.bert_models.classifier_model(
 bert_config, num_labels=1 )
credit = pd.read_csv('.. /input/credit_card_balance.csv' ).replace({365243: np.nan})
 credit = convert_types(credit)
 credit['OVER_LIMIT'] = credit['AMT_BALANCE'] > credit['AMT_CREDIT_LIMIT_ACTUAL']
credit['BALANCE_CLEARED'] = credit['AMT_BALANCE'] == 0.0
credit['LOW_PAYMENT'] = credit['AMT_PAYMENT_CURRENT'] < credit['AMT_INST_MIN_REGULARITY']
credit['LATE'] = credit['SK_DPD'] > 0.0<concatenate>
data_batch = {key: val[:10] for key, val in data_train.items() }
bert_classifier(
 data_batch, training=True
).numpy()
credit_info = agg_grandchild(credit, previous, 'SK_ID_PREV', 'SK_ID_CURR', 'CC')
 del credit, previous
credit_info.shape<set_options>
checkpoint = tf.train.Checkpoint(model=bert_encoder)
 checkpoint.restore(
 os.path.join(gs_folder_bert, 'bert_model.ckpt')).assert_consumed()
gc.collect()
gc.enable()<merge>
epochs = 3
batch_size = 32
eval_batch_size = 32
train_data_size = len(data_train_labels)
 steps_per_epoch = int(train_data_size / batch_size)
 num_train_steps = steps_per_epoch * epochs
warmup_steps = int(epochs * train_data_size * 0.1 / batch_size)
 optimizer = nlp.optimization.create_optimizer(
 2e-5, num_train_steps=num_train_steps, num_warmup_steps=warmup_steps )
time.sleep(600)
 app = app.merge(credit_info, on = 'SK_ID_CURR', how = 'left')
 del credit_info
app.shape<train_model>
metrics = [tf.keras.metrics.BinaryAccuracy('binary_accuracy', dtype=tf.float32)]
loss = tf.keras.losses.BinaryCrossentropy(from_logits=True,name='binary_crossentropy')
 bert_classifier.compile(
 optimizer=optimizer,
loss=loss,
metrics=metrics)
print('After manual feature engineering, there are {} features.'.format(app.shape[1] - 2))<set_options>
bert_classifier.fit(
 data_train, data_train_labels,
validation_data=(data_validation, data_validation_labels),
batch_size=32,
epochs=epochs )
gc.enable()
gc.collect()<save_to_csv>
y_pred = bert_classifier.predict(data_test )
app.to_csv('clean_manual_features.csv', chunksize = 100 )<split>
result = bert_classifier(data_test, training=False)
app.reset_index(inplace = True)
 train, test = app[app['TARGET'].notnull() ].copy() , app[app['TARGET'].isnull() ].copy()
gc.enable()
del app
gc.collect()<init_hyperparams>
check = [1 if i>0 else 0 for i in result]
params = {'is_unbalance': True,
'n_estimators': 2673,
'num_leaves': 77,
'learning_rate': 0.00764,
'min_child_samples': 460,
'boosting_type': 'gbdt',
'subsample_for_bin': 240000,
'reg_lambda': 0.20,
'reg_alpha': 0.88,
'subsample': 0.95,
'colsample_bytree': 0.7}<train_model>
print(classification_report(check, data_test_labels,))
train_labels = np.array(train.pop('TARGET')).reshape(( -1,))
test_ids = list(test.pop('SK_ID_CURR'))
test = test.drop(columns = ['TARGET'])
 train = train.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )<train_model>
export_dir='./saved_model'
tf.saved_model.save(bert_classifier, export_dir=export_dir )
model = lgb.LGBMClassifier(**params)
 model.fit(train, train_labels )<save_to_csv>
export_dir='./saved_model'
clf = tf.saved_model.load(export_dir )
preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids,
'TARGET': preds})
 submission['SK_ID_CURR'] = submission['SK_ID_CURR'].astype(int)
 submission['TARGET'] = submission['TARGET'].astype(float)
 submission.to_csv('submission_manual.csv', index = False )<create_dataframe>
sub = pd.read_csv(test )
features = list(train.columns)
 fi = pd.DataFrame({'feature': features,
'importance': model.feature_importances_} )<load_from_csv>
data_sub = bert_encode(sub, tokenizer )
app_train = pd.read_csv(path + "application_train.csv")
 app_train.head() 
<load_from_csv>
sub_val = clf([data_sub['input_word_ids'],
data_sub['input_mask'],
data_sub['input_type_ids']], training=False)
bureau = pd.read_csv(path + "bureau.csv")
 bureau['YEAR']=(( bureau['DAYS_CREDIT'] /365)).abs().pow(0.5 ).round(0)
 bureau.head() 
<feature_engineering>
target = [1 if i>0 else 0 for i in sub_val]
<load_from_csv><EOS> 
my_submission = pd.DataFrame({'id': sub.id, 'Target': target})
 my_submission.to_csv('submission.csv', index=False )