Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<predict_on_test>
debug = False
warnings.simplefilter('ignore')
 plt.style.use('seaborn')
 random_state = 333
dt.score(X_test,y_test)
 y_pred=dt.predict(X_test )<compute_test_metric>
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv', index_col='ID_code')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv', index_col='ID_code')
 public = np.load('.. /input/helpingdata/public_LB.npy')
 private = np.load('.. /input/helpingdata/private_LB.npy')
 fake = np.load('.. /input/helpingdata/synthetic_samples_indexes.npy')
 target = train_df['target']
confusion_matrix(y_test,y_pred )<load_from_csv>
train_df, test_df = reverse(train_df, test_df )
test=pd.read_csv('/kaggle/input/titanic/test.csv')
 test_dummies=pd.get_dummies(data=test,columns=['Sex','Embarked'],drop_first=True)
 test_dummies['Fare'].fillna(test_dummies['Fare'].dropna().median() ,inplace=True)
 test_dummies['Age'].fillna(test_dummies['Age'].dropna().median() ,inplace=True)
 test_dummies.info()<predict_on_test>
real_idx = np.sort(np.concatenate(( public, private)))
 real_idx
X_testf=test_dummies.loc[:,['Sex_male','SibSp','Parch','Pclass','Fare','Embarked_S','Embarked_Q','Age','Fare']]
predictions=dt.predict(X_testf )<prepare_output>
test_real = test_df.iloc[real_idx]
test_real.head()
Id=test_dummies['PassengerId']
sub_df=pd.DataFrame({'PassengerId':Id,'Survived':predictions})
 sub_df.head()<save_to_csv>
%%time
concat_df = pd.concat(( train_df, test_real))
features = train_df.columns[1:]
if debug:
features = features[:3]
for i, feature in enumerate(features):
print('Calculating var_{}'.format(i), end='\r')
 n_dups_dict = concat_df.loc[:, feature].value_counts().to_dict()
train_df.loc[:, 'count_{}'.format(feature)] = [n_dups_dict.get(value, 1)* np.sign(value)for value in train_df.loc[:, feature]]
test_df.loc[:, 'count_{}'.format(feature)] = [n_dups_dict.get(value, 1)* np.sign(value)for value in test_df.loc[:, feature]]
train_df.loc[:, '{}_x_count_{}'.format(feature, feature)] = train_df.loc[:, feature] * np.absolute(train_df.loc[:, 'count_{}'.format(feature)])
 test_df.loc[:, '{}_x_count_{}'.format(feature, feature)] = test_df.loc[:, feature] * np.absolute(test_df.loc[:, 'count_{}'.format(feature)])
 train_df.loc[:, 'count_inverse_{}'.format(feature)] = [(1 / n_dups_dict.get(value, 1)) * np.sign(value)for value in train_df.loc[:, feature]]
test_df.loc[:, 'count_inverse_{}'.format(feature)] = [(1 / n_dups_dict.get(value, 1)) * np.sign(value)for value in test_df.loc[:, feature]]
train_df.loc[:, '{}_x_count_inverse_{}'.format(feature, feature)] = train_df.loc[:, feature] * np.absolute(train_df.loc[:, 'count_inverse_{}'.format(feature)])
 test_df.loc[:, '{}_x_count_inverse_{}'.format(feature, feature)] = test_df.loc[:, feature] * np.absolute(test_df.loc[:, 'count_inverse_{}'.format(feature)])
 print(train_df.shape )
sub_df.to_csv('submission.csv',index=False )<set_options>
gc.collect()
plt.style.use('fivethirtyeight')
<load_from_csv>
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 7,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 1,
"feature_fraction" : 0.3,
"min_data_in_leaf": 80,
"min_sum_hessian_in_leaf" : 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}
sample = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv")
 df_train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<feature_engineering>
%%time
df_train = train_df.reset_index()
df_test = test_df.reset_index()
if debug:
df_train = df_train[:1000]
df_test = df_test[:1000]
target = target[:1000]
skf = StratifiedKFold(n_splits = 5, shuffle=True, random_state=random_state)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, target)) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
print("shape of training data: ", X_train.shape)
 trn_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=1000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid = lgb_clf.predict(X_valid, num_iteration=lgb_clf.best_iteration)
 yp = lgb_clf.predict(X_test, num_iteration=lgb_clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = [col for col in X_train.columns]
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp
df_train["log_SalePrice"] = np.log10(df_train["SalePrice"] )<feature_engineering>
gc.collect()
dtype = pd.DataFrame({"columns":df_train.dtypes.index,
"dtype":df_train.dtypes})
 dtype["dtype"] = [str(i)for i in dtype["dtype"]]
num_columns = dtype.query('dtype=="int64" | dtype=="float64"')["columns"].values[1:-2]
obj_columns = dtype.query('dtype=="object"')["columns"].values
print("numerical_values_count:{}".format(len(num_columns)))
 print("object_values_count:{}".format(len(obj_columns)) )<prepare_x_and_y>
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.5f, std: %.5f.All auc: %.5f." %(mean_auc, std_auc, all_auc))
y = df_train["log_SalePrice"]
df_num_train = df_train[num_columns]
df_num_test = df_test[num_columns]
df_cate_train = df_train[obj_columns]
df_cate_test = df_test[obj_columns]<categorify>
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )