Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<count_values>
%matplotlib inline
SEED = 50
print(os.listdir(".. /input"))
print(df.MSZoning.value_counts())
 df[df.MSZoning.isnull() ]<groupby>
dt_train = pd.read_csv(".. /input/train.csv")
 print("Number of Training Examples = {}".format(dt_train.shape))
df.groupby("Neighborhood" ).MSZoning.value_counts()<feature_engineering>
dt_test = pd.read_csv(".. /input/test.csv")
 print("Number of Test Examples = {}".format(dt_test.shape))
df.loc[(df.MSZoning.isnull())&(df.Neighborhood == "IDOTRR"), "MSZoning"] = "C(all)"
df.loc[(df.MSZoning.isnull())&(df.Neighborhood == "Mitchel"), "MSZoning"] = "RL"<count_values>
dt_train.name = "Training Dataset"
dt_test.name = "Test Dataset"
for dt in [dt_train, dt_test]:
print('{} have {} missing values.'.format(dt.name, int(dt.isnull().values.any())) )
df.KitchenQual.value_counts()<count_values>
ones = dt_train['target'].value_counts() [1]
zeros = dt_train['target'].value_counts() [0]
ones_per = ones / dt_train.shape[0] * 100
zeros_per = zeros / dt_train.shape[0] * 100
print('{} out of {} rows are Class 1 and it is the {:.2f}% of the dataset.'.format(ones, dt_train.shape[0], ones_per))
print('{} out of {} rows are Class 0 and it is the {:.2f}% of the dataset.'.format(zeros, dt_train.shape[0], zeros_per))
print(df.groupby(["OverallQual", "KitchenAbvGr"] ).KitchenQual.value_counts())
 df.loc[(df.KitchenQual.isnull())&(df.OverallQual == 5)&(df.KitchenAbvGr == 1), "KitchenQual"] = "TA"<count_values>
dt_train_unique = dt_train.agg(['nunique'] ).transpose().sort_values(by='nunique')
 dt_test_unique = dt_test.agg(['nunique'] ).transpose().sort_values(by='nunique')
 dt_uniques = dt_train_unique.drop('target' ).reset_index().merge(dt_test_unique.reset_index() , how='right', right_index=True, left_index=True)
 print(dt_uniques.head())
print(df.SaleType.value_counts())
 df[df.SaleType.isnull() ]<count_values>
dt_uniques.drop(columns=['index_y'], inplace=True)
 dt_uniques.columns = ['Feature', 'Training Set Unique Count', 'Test Set Unique Count']
print(dt_uniques.head() )
print(df.groupby(["Neighborhood", "SaleCondition"] ).SaleType.value_counts())
 df.loc[(df.SaleType.isnull())&(df.Neighborhood == "Sawyer")&(df.SaleCondition == "Normal"), "SaleType"] = "WD"<count_values>
cols=["target","ID_code"]
X = dt_train.drop(cols, axis=1)
 y = dt_train.target
print(df.Electrical.value_counts())
 df[df.Electrical.isnull() ]<count_values>
params = {'objective': "binary",
'boost': "gbdt",
'boost_from_average': "false",
'num_threads': 8,
'learning rate': 0.001,
'num_leaves': 4,
'max_depth': 8,
'tree_learner': "serial",
'feature_fraction': 0.05,
'bagging_fraq': 1.0,
'bagging_fraction': 0.4,
'min_data_in_leaf': 80,
'verbosity': 1,
'metric': 'auc',
'sigmoid':1.0,
'lambda_l1':0.01,
'snapshot_freq':1
}
df[(df.YearBuilt > 2005)].Electrical.value_counts()<correct_missing_values>
folds = StratifiedKFold(n_splits=5, shuffle=False, random_state=10)
 length_train = np.zeros(len(dt_train))
dt_predictions = np.zeros(len(dt_test))
important_dt = pd.DataFrame()
df.Electrical.fillna("SBrkr", inplace=True )<count_values>
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
print("folds {}," "fold_n {}," "started at {}".format(folds, fold_n, time.ctime()))
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params, train_data, num_boost_round=2000,
valid_sets = [train_data, valid_data], verbose_eval=1000,
early_stopping_rounds = 2500,)
 length_train[valid_index] = lgb_model.predict(X_valid, num_iteration=lgb_model.best_iteration)
print(df.Exterior1st.value_counts())
 df[df.Exterior1st.isnull() ]<count_values>
dt_tes = dt_test.drop(['ID_code'], axis=1)
 print(dt_tes.head() )
print(df[(df.RoofMatl == "Tar&Grv")].Exterior1st.value_counts())
 df.Exterior1st.fillna("Plywood", inplace=True )<count_values>
dt_predictions += lgb_model.predict(dt_tes, num_iteration = lgb_model.best_iteration)/folds.n_splits
print("CV Score: {:<8.5f}".format(roc_auc_score(y, length_train)) )
print(df[(df.RoofMatl == "Tar&Grv")].Exterior2nd.value_counts())
 df.Exterior2nd.fillna("Plywood", inplace=True )<count_values>
dt_predictio = pd.DataFrame(dt_predictions )
print(df.Functional.value_counts())
 df[df.Functional.isnull() ]<count_values>
submit = pd.DataFrame({"ID_code": dt_test["ID_code"].values})
 submit["target"] = dt_predictio
submit.to_csv("submit.csv", index=False )
<correct_missing_values><EOS> 
new = pd.read_csv("submit.csv" )