Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<choose_model_class>
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 9999
def build_model(embedding_matrix, num_aux_targets, lr=0.0, lr_d=0.0, dr=0.0):
word_input = Input(shape=(WORD_MAX_LEN,), name="word_input")
 word_embed = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(word_input)
 embed = word_embed
embed = SpatialDropout1D(dr )(embed)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(embed)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 hidden = Concatenate()([avg_pool1, max_pool1])
 hidden = Add()([hidden, Dense(hidden.shape[-1].value, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid', name="result" )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid', name="aux" )(hidden)
 model = Model(inputs=[word_input], outputs=[result, aux_result])
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr=lr, decay=lr_d), metrics=["acc"])
 return model<concatenate>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train rows and columns : ", train_df.shape)
 print("Test rows and columns : ", test_df.shape )
%%time
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<define_variables>
missing_df = train_df.isnull().sum(axis=0 ).reset_index()
missing_df.columns = ['column_name', 'missing_count']
missing_df = missing_df[missing_df['missing_count']>0]
missing_df = missing_df.sort_values(by='missing_count')
 missing_df
train_gen = Generator(x_train, y_train, y_aux_train, batch_size=BATCH_SIZE )<string_transform>
dtype_df = train_df.dtypes.reset_index()
dtype_df.columns = ["Count", "Column Type"]
dtype_df.groupby("Column Type" ).aggregate('count' ).reset_index()
word_x_test = tokenizer.texts_to_sequences(x_test)
 word_x_test = sequence.pad_sequences(word_x_test, maxlen=WORD_MAX_LEN )<train_model>
unique_df = train_df.nunique().reset_index()
unique_df.columns = ["col_name", "unique_count"]
constant_df = unique_df[unique_df["unique_count"]==1]
constant_df.shape
checkpoint_predictions = []
for i in range(EPOCHS):
model.fit_generator(train_gen, steps_per_epoch=train_gen.__len__() , epochs=1)
 p = model.predict([word_x_test], batch_size=2048, verbose=1)[0].flatten()
checkpoint_predictions.append(p )<groupby>
train_X = train_df.drop(constant_df.col_name.tolist() + ["ID", "target"], axis=1)
 test_X = test_df.drop(constant_df.col_name.tolist() + ["ID"], axis=1)
 train_y = np.log1p(train_df["target"].values )
lstm_pred = np.average(checkpoint_predictions, axis=0 )<save_to_csv>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"learning_rate" : 0.01,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=200, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result
predictions = lstm_pred
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<string_transform>
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 pred_test_full = 0
for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index,:], train_X.loc[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)
 pred_test_full += pred_test
pred_test_full /= 5.
 pred_test_full = np.expm1(pred_test_full )
<define_variables><EOS> 
sub_df = pd.DataFrame({"ID":test_df["ID"].values})
 sub_df["target"] = pred_test_full
sub_df.to_csv("baseline_lgb.csv", index=False )