Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<categorify>
notebookstart= time.time()
random.seed(2018)
 get_ipython().magic('matplotlib inline')
 id_col = "ID"
target_var = "target"
Debug = False
Home = False
Build_Results_csv = False
results = pd.DataFrame(columns = ["Rounds","Score","STDV", "LB", "Parameters"])
 if Build_Results_csv is True & Home is True: results.to_csv("results.csv")
 if Home is True:
path = r"D:\My Computer\DATA\Santander"
os.chdir(path)
 print("Data Load Stage")
 training = pd.read_csv('train.csv', index_col = id_col)
 if Debug is True : training = training.sample(100)
 traindex = training.index
testing = pd.read_csv('test.csv', index_col = id_col)
 if Debug is True : testing = testing.sample(100)
 testdex = testing.index
else:
print("Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = id_col)
 if Debug is True : training = training.sample(100)
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = id_col)
 if Debug is True : testing = testing.sample(100)
 testdex = testing.index
y = np.log1p(training[target_var])
 training.drop(target_var,axis=1, inplace=True)
 print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
print("Combine Train and Test")
 df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))
train_input = bert_encode(df_train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(df_test.text.values, tokenizer, max_len=160)
 train_labels = df_train.target.values<choose_model_class>
test_df = df.loc[testdex,:]
vocab = df.columns
lgtrain = lgb.Dataset(df.loc[traindex,vocab],y ,feature_name = "auto")
 print("Starting LightGBM.Train shape: {}, Test shape: {}".format(df.loc[testdex,:].shape,test_df.shape))
print("Feature Num: ",len(vocab))
del df; gc.collect() ;
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<train_model>
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
"learning_rate": 0.01,
"num_leaves": 180,
"feature_fraction": 0.50,
"bagging_fraction": 0.50,
'bagging_freq': 4,
"max_depth": -1,
"reg_alpha": 0.3,
"reg_lambda": 0.1,
"min_child_weight":10,
'zero_as_missing':True
}
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=2,
callbacks=[checkpoint],
batch_size=16
)<predict_on_test>
modelstart= time.time()
lgb_cv = lgb.cv(
 params = lgbm_params,
train_set = lgtrain,
num_boost_round=2000,
stratified=False,
nfold = 5,
verbose_eval=50,
seed = 23,
early_stopping_rounds=75)
 optimal_rounds = np.argmin(lgb_cv['rmse-mean'])
 best_cv_score = min(lgb_cv['rmse-mean'])
 print("
Optimal Round: {}
Optimal Score: {} + {}".format(
 optimal_rounds,best_cv_score,lgb_cv['rmse-stdv'][optimal_rounds]))
results = results.append({"Rounds": optimal_rounds,
"Score": best_cv_score,
"STDV": lgb_cv['rmse-stdv'][optimal_rounds],
"LB": None,
"Parameters": lgbm_params}, ignore_index=True)
 if Home is True:
with open('results.csv', 'a')as f:
results.to_csv(f, header=False )
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
pd.set_option('max_colwidth', 800)
 display(results.sort_values(by="Score",ascending = True))
df_test['target'] = test_pred.round().astype(int)
 submission = df_test[['id', 'target']]
submission.to_csv('bert_submission.csv', index=False )<feature_engineering>
learning_rates = [0.012,0.008]
for param in learning_rates:
print("Learning Rate: ", param)
 modelstart= time.time()
lgbm_params["learning_rate"] = param
lgb_cv = lgb.cv(
 params = lgbm_params,
train_set = lgtrain,
num_boost_round=10000,
stratified=False,
nfold = 5,
verbose_eval=200,
seed = 23,
early_stopping_rounds=75)
 optimal_rounds = np.argmin(lgb_cv['rmse-mean'])
 best_cv_score = min(lgb_cv['rmse-mean'])
 print("Optimal Round: {}
Optimal Score: {} + {}".format(
 optimal_rounds,best_cv_score,lgb_cv['rmse-stdv'][optimal_rounds]))
print("
results = results.append({"Rounds": optimal_rounds,
"Score": best_cv_score,
"STDV": lgb_cv['rmse-stdv'][optimal_rounds],
"LB": None,
"Parameters": lgbm_params}, ignore_index=True)
 if Home is True:
with open('results.csv', 'a')as f:
results.to_csv(f, header=False)
def prepare_data(df):
df.keyword.fillna("unknown", inplace = True)
 df.location.fillna("unknown", inplace = True)
 df['clean_tweet'] = df.text + ' location: ' + df.location + ' keyword: ' + df.keyword 
<load_from_csv>
pd.set_option('max_colwidth', 800)
 display(results.sort_values(by="Score",ascending = True))
df = pd.read_csv(os.path.join('/kaggle/input/nlp-getting-started', 'train.csv'))
df = df.sample(frac = 1)
 prepare_data(df)
 train_df, val_df = np.split(df, [int (.9*len(df)) ] )<define_variables>
final_model_params = results.iloc[results["Score"].idxmin() ,:]["Parameters"]
optimal_rounds = results.iloc[results["Score"].idxmin() ,:]["Rounds"]
print("Parameters for Final Models:
",final_model_params)
 print("Score: {} +/- {}".format(results.iloc[results["Score"].idxmin() ,:]["Score"],results.iloc[results["Score"].idxmin() ,:]["STDV"]))
print("Rounds: ", optimal_rounds )
training_args = TrainingArguments(
 output_dir= '/kaggle/output',
num_train_epochs=3,
per_device_train_batch_size=16,
per_device_eval_batch_size=64,
warmup_steps=500,
weight_decay=0.01,
logging_dir='/kaggle/output',
logging_steps=10, 
)
<load_pretrained>
allmodelstart= time.time()
multi_seed_pred = dict()
all_feature_importance_df = pd.DataFrame()
all_seeds = [27,22,300,401]
for seeds_x in all_seeds:
modelstart= time.time()
print("Seed: ", seeds_x,)
 final_model_params["seed"] = seeds_x
lgb_reg = lgb.train(
 final_model_params,
lgtrain,
num_boost_round = optimal_rounds + 1,
verbose_eval=200)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = vocab
fold_importance_df["importance"] = lgb_reg.feature_importance()
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)
 multi_seed_pred[seeds_x] = list(lgb_reg.predict(test_df))
print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
print("
del lgb_reg
cols = all_feature_importance_df[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = all_feature_importance_df.loc[all_feature_importance_df.feature.isin(cols)]
plt.figure(figsize=(8,10))
sns.barplot(x="importance", y="feature",
data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances.png')
 print("All Model Runtime: %0.2f Minutes"%(( time.time() - allmodelstart)/60))
sub_preds = pd.DataFrame.from_dict(multi_seed_pred ).replace(0,0.000001)
 del multi_seed_pred; gc.collect() ;
f, ax = plt.subplots(figsize=[8,6])
 sns.heatmap(sub_preds.corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Percentage %'},cmap="plasma",ax=ax)
 ax.set_title("Correlation Plot for Seed Diversified Models")
 plt.show()
model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased" )<categorify>
mean_sub = np.expm1(sub_preds.mean(axis=1 ).rename(target_var))
mean_sub.index = testdex
mean_sub.to_csv('mean_sub_ep{}_sc{}.csv'.format(optimal_rounds,round(best_cv_score,5))
,index = True, header=True)
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))
mean_sub.head()