Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<train_model>
warnings.filterwarnings("ignore")
 init_notebook_mode(connected=True)
class PyTorch(Estimator):
def fit(self, x_train, y_train, x_valid, y_valid):
train_tensor = TensorDataset(
 torch.from_numpy(x_train.astype('float32')) ,
torch.from_numpy(y_train.astype('float32')))
 train_loader = DataLoader(train_tensor, batch_size=256, shuffle=True)
 self.model = nn.Sequential(
 weight_norm(nn.Linear(x_train.shape[1], 128)) ,
nn.ReLU() ,
weight_norm(nn.Linear(128, 128)) ,
nn.ReLU() ,
weight_norm(nn.Linear(128, 128)) ,
nn.ReLU() ,
weight_norm(nn.Linear(128, 128)) ,
nn.ReLU() ,
weight_norm(nn.Linear(128, 1)) ).to(device)
 for m in self.model:
if isinstance(m, nn.Linear):
nn.init.kaiming_normal_(m.weight_v)
 nn.init.kaiming_normal_(m.weight_g)
 nn.init.constant_(m.bias, 0)
 criterion = nn.L1Loss()
optimizer = torch.optim.Adam(self.model.parameters() , betas=(0.9, 0.999), lr=1e-3)
 self.model.train()
n_epochs = 50
for epoch in range(n_epochs):
epoch_loss = 0.0
for train_part, y_part in train_loader:
optimizer.zero_grad()
y_pred_part = self.model(train_part.to(device))
loss = criterion(y_pred_part.reshape(-1), y_part.to(device))
loss.backward()
optimizer.step()
epoch_loss += y_pred_part.shape[0] * loss.item()
print('Epoch %3d / %3d.Loss = %.5f' %(epoch + 1, n_epochs, epoch_loss / x_train.shape[0]))
def predict(self, x):
self.model.eval()
x_tensor = torch.from_numpy(x.astype('float32'))
x_loader = DataLoader(x_tensor, batch_size=256, shuffle=False)
 y_pred = np.empty(0)
 with torch.no_grad() :
for x_part in x_loader:
y_pred_part = self.model(x_part.to(device)).data.cpu().numpy().reshape(-1)
 y_pred = np.append(y_pred, y_pred_part)
 return y_pred<train_model>
%%time
sctp_train = pd.read_csv('.. /input/train.csv')
 sctp_test = pd.read_csv('.. /input/test.csv')
 print("=="*45)
 print("The training dataset has {0} rows and {1} columns".format(sctp_train.shape[0], sctp_train.shape[1]))
print("The testing dataset has {0} rows and {1} columns".format(sctp_test.shape[0], sctp_test.shape[1]))
print("=="*45 )
%%time
scaler = StandardScaler()
oof, trained_estimators = pipeline_fit(PyTorch() , df_train, scaler )<set_options>
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("Your selected dataframe has " + str(df.shape[1])+ " columns.
 "
"There are " + str(mis_val_table_ren_columns.shape[0])+
" columns that have missing values.")
 return mis_val_table_ren_columns
missing_values_table(sctp_train )
del df_train
gc.collect()<load_from_csv>
corr_df_sel = corr_df.ix[(corr_df['corr_values'] < -0.05)|(corr_df['corr_values']>0.05)]
corr_df_sel
df_test = pd.read_csv('.. /input/test_V2.csv', index_col = 'Id')
 df_test.shape<feature_engineering>
X = sctp_train.drop(["ID_code", "target"], axis=1)
 Y = sctp_train["target"]
test_X = sctp_test.drop(["ID_code"], axis=1)
 X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size=0.2, random_state=1)
 print(X_train.shape, y_train.shape)
 print(X_val.shape, y_val.shape )
df_test = reduce_mem_usage(df_test )<create_dataframe>
rfc_model = RandomForestClassifier(random_state=0 ).fit(X_train, y_train)
 perm = PermutationImportance(rfc_model, random_state=1 ).fit(X_val, y_val)
 eli5.show_weights(perm, feature_names = X_val.columns.tolist() )
df_test_id = pd.DataFrame(index=df_test.index )<set_options>
sm = SMOTE(random_state=42)
 X_resamp_tr, y_resamp_tr = sm.fit_resample(X, Y)
 print('Resampled dataset shape %s' % Counter(y_resamp_tr))
X_resamp_tr = pd.DataFrame(X_resamp_tr)
 y_resamp_tr = pd.DataFrame({"target": y_resamp_tr} )
gc.collect()<predict_on_test>
params = {'num_leaves': 9,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 16,
'learning_rate': 0.0123,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': 11,
'reg_alpha': 1.728910519108444,
'reg_lambda': 4.9847051755586085,
'random_state': 42,
'metric': 'auc',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01077313523861969,
'min_child_weight': 19.428902804238373,
'num_threads': 4}
y = pipeline_predict(trained_estimators, df_test, scaler )<set_options>
%%time
fold_n=3
folds = StratifiedKFold(n_splits=fold_n, shuffle=True, random_state=30)
 y_pred_lgb = np.zeros(len(test_X))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X_resamp_tr,y_resamp_tr)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X_resamp_tr.iloc[train_index], X_resamp_tr.iloc[valid_index]
y_train, y_valid = y_resamp_tr.iloc[train_index], y_resamp_tr.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 y_pred_lgb += lgb_model.predict(test_X, num_iteration=lgb_model.best_iteration)/3
del df_test
gc.collect()<save_to_csv>
submission_lgb_smote_2 = pd.DataFrame({
"ID_code": sctp_test["ID_code"],
"target": y_pred_lgb
})
 submission_lgb_smote_2.to_csv('submission_lgb_smote_2.csv', index=False )