Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<categorify>
def add_columns(df):
for col in df.columns:
df[col] =(( df[col] - df[col].mean())/ df[col].std() ).astype('float32')
 df[col+'_s'] = df[col] * df[col]
df[col+'_c'] = df[col] * df[col] * df[col]
df[col+'_q'] = df[col] * df[col] * df[col] * df[col]
df[col+'_r'] = rankdata(df[col] ).astype('float32')
 df[col+'_n'] = norm.cdf(df[col] ).astype('float32')
 for col in df.columns:
df[col] =(( df[col] - df[col].mean())/ df[col].std() ).astype('float32')
df["LotFrontage"] = df["LotFrontage"].fillna(df.groupby(["Neighborhood", "LotShape", "LotConfig"])["LotFrontage"].transform("median"))
df["LotFrontage"] = df["LotFrontage"].fillna(df.groupby(["Neighborhood", "LotShape"])["LotFrontage"].transform("median"))
df["LotFrontage"] = df["LotFrontage"].fillna(df.groupby("Neighborhood")["LotFrontage"].transform("median"))<correct_missing_values>
pd.set_option('display.max_columns', 200 )
df.Utilities.fillna(df.Utilities.mode() [0], inplace=True)
 missing(df )<data_type_conversions>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
df["MSSubClass"] = df["MSSubClass"].astype("str" )<feature_engineering>
target = train_df.target
ID_code = test_df.ID_code
train_df = train_df.drop('target',axis = 1)
 train_df = train_df.drop('ID_code',axis=1)
 test_df = test_df.drop('ID_code',axis=1 )
df.loc[(df.GarageYrBlt == 2207), "GarageYrBlt"] = 2007
df.loc[(df.Exterior2nd == "CmentBd"), "Exterior2nd"] = "CemntBd"
df.loc[(df.Exterior2nd == "Wd Shng"), "Exterior2nd"] = "WdShing"
df.loc[(df.Exterior2nd == "Brk Cmn"), "Exterior2nd"] = "BrkComm"<create_dataframe>
pca = PCA(n_components=1, copy=True, random_state=4)
 pca.fit(pandas.concat([train_df,test_df]))
P1 = pca.transform(train_df)
 P2 = pca.transform(test_df)
dff = df.copy()<count_values>
len_train = len(train_df)
 df = pd.concat([train_df,test_df])
 add_columns(df)
 train_df = df[:len_train]
test_df = df[len_train:]
del df
num_cols, cat_cols, ord_cols = col_types(dff)
 for col in dff.columns:
print("For column: ", col + "
")
 print(dff[col].value_counts() , "
" )<feature_engineering>
train_df['P1'] = P1[:,0]
test_df['P1'] = P2[:,0]
dff["Older1945"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["30", "70"] else 0)
 dff["Newer1946"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["20", "60", "120", "160"] else 0)
 dff["AllStyles"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["20", "90", "190"] else 0)
 dff["AllAges"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["40", "45", "50", "75", "90", "150", "190"] else 0)
 dff["Pud"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["120", "150", "160", "180"] else 0)
 dff["Split"] = dff["MSSubClass"].apply(lambda x: 1 if x in ["80", "85""180"] else 0)
 dff["MSSubClass"] = dff["MSSubClass"].apply(lambda x: "180" if x == "150" else x )<feature_engineering>
bayesian_tr_index, bayesian_val_index = list(StratifiedKFold(n_splits=2, shuffle=True, random_state=1 ).split(train_df, target)) [0]
dff["MSZoning"] = dff["MSZoning"].apply(lambda x: "R" if x.startswith("R")else x )<feature_engineering>
params={'feature_fraction': 0.05303293150022274,
'lambda_l1': 4.742447900648306,
'lambda_l2': 3.797046693025151,
'learning_rate': 0.012061342816038765,
'max_depth': 14.202377588960745,
'min_data_in_leaf': 9.784931229481742,
'min_gain_to_split': 0.06340137767250764,
'min_sum_hessian_in_leaf': 0.00672958286082188,
'num_leaves': 14.036858809140814}
dff["North"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["Blmngtn", "BrDale", "ClearCr", "Gilbert", "Names", "NoRidge",
"NPkVill", "NWAmes", "NoRidge", "NridgHt", "Sawyer", "Somerst",
"StoneBr", "Veenker", "NridgHt"] else 0)
 dff["South"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["Blueste", "Edwards", "Mitchel", "MeadowV",
"SWISU", "IDOTRR", "Timber"] else 0)
 dff["Downtown"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["BrkSide", "Crawfor", "OldTown", "CollgCr"] else 0)
 dff["East"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["IDOTRR", "Mitchel"] else 0)
 dff["West"] = dff["Neighborhood"].apply(lambda x: 1 if x in ["Edwards", "NWAmes", "SWISU", "Sawyer", "SawyerW"] else 0 )<feature_engineering>
param_lgb = {
'num_leaves': int(params['num_leaves']),
'max_bin': 63,
'min_data_in_leaf': int(params['min_data_in_leaf']),
'learning_rate': params['learning_rate'],
'min_sum_hessian_in_leaf': params['min_sum_hessian_in_leaf'],
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': params['feature_fraction'],
'lambda_l1': params['lambda_l1'],
'lambda_l2': params['lambda_l2'],
'min_gain_to_split': params['min_gain_to_split'],
'max_depth': int(params['max_depth']),
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
}
dff.loc[(dff["Condition1"] == "Feedr")|(dff["Condition2"] == "Feedr"), "StreetDegree"] = 1
dff.loc[(dff["Condition1"] == "Artery")|(dff["Condition2"] == "Artery"), "StreetDegree"] = 2
dff["StreetDegree"].fillna(0, inplace = True)
 dff.loc[(dff["Condition1"].isin(["RRNn", "RRNe"])) |(dff["Condition2"].isin(["RRNn", "RRNe"])) , "RailroadDegree"] = 1
dff.loc[(dff["Condition1"].isin(["RRAn", "RRAe"])) |(dff["Condition2"].isin(["RRAn", "RRAe"])) , "RailroadDegree"] = 2
dff["RailroadDegree"].fillna(0, inplace = True)
 dff.loc[(dff["Condition1"] == "PosN")|(dff["Condition2"] == "PosN"), "OffsiteFeature"] = 1
dff.loc[(dff["Condition1"] == "PosA")|(dff["Condition2"] == "PosA"), "OffsiteFeature"] = 2
dff["OffsiteFeature"].fillna(0, inplace = True)
 dff["Norm1"] = dff["Condition1"].apply(lambda x: 1 if x == "Norm" else 0)
 dff["Norm2"] = dff["Condition2"].apply(lambda x: 1 if x == "Norm" else 0)
 dff["Norm"] = dff["Norm1"] + dff["Norm2"]
dff.drop(["Norm1", "Norm2"], axis = 1, inplace = True )<define_variables>
nfold = 5
lotshape = {"IR3": 1, "IR2": 2, "IR1": 3, "Reg": 4}
landcontour = {"Low":1, "HLS": 2, "Bnk":3, "Lvl": 4}
utilities = {"ELO": 1, "NoSeWa": 2, "NoSewr": 3, "AllPub": 4}
landslope = {"Sev": 1, "Mod": 2, "Gtl": 3}
general = {"None": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5}
bsmtexposure = {"None": 0, "No": 0, "Mn": 1, "Av": 2, "Gd": 3}
bsmtfintype = {"None": 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6}
electrical = {"Mix": 1, "FuseP": 2, "FuseF": 3, "FuseA": 4, "SBrkr": 5}
functional = {"Typ": 1, "Min1": 2, "Min2": 3, "Mod": 4, "Maj1": 5, "Maj2": 6, "Sev": 7, "Sal": 8}
garagefinish = {"None": 0, "Unf": 1, "RFn": 2, "Fin": 3}
fence = {"None": 0, "MnWw": 1, "GdWo": 2, "MnPrv": 3, "GdPrv": 4}
dff.replace({"LotShape": lotshape, "LandContour": landcontour, "Utilities": utilities, "LandSlope": landslope,
"BsmtExposure": bsmtexposure, "BsmtFinType1": bsmtfintype, "BsmtFinType2":bsmtfintype, "Electrical": electrical,
"Functional": functional, "GarageFinish": garagefinish, "Fence": fence},
inplace = True)
 for col in ["ExterQual", "ExterCond", "BsmtQual", "BsmtCond", "HeatingQC", "KitchenQual",
"FireplaceQu", "GarageQual", "GarageCond", "PoolQC"]:
dff[col] = dff[col].replace(general )<feature_engineering>
gc.collect()
dff["BldgType"] = dff["BldgType"].apply(lambda x: "2Fam" if x in ["2fmCon", "Duplex"] else x)
 dff["SaleType"] = dff["SaleType"].apply(lambda x: "WD" if x.endswith("WD")else x)
 dff["SaleType"] = dff["SaleType"].apply(lambda x: "Contract" if x.startswith("Con")else x)
 dff["SaleType"] = dff["SaleType"].apply(lambda x: "Oth" if x == "COD" else x)
 dff["SaleCondition"] = dff["SaleCondition"].apply(lambda x: "Abnormal_Adjland" if x in ["Abnorml", "AdjLand"] else x)
 dff["SaleCondition"] = dff["SaleCondition"].apply(lambda x: "Alloca_Family" if x in ["Alloca", "Family"] else x)
 dff["SaleCondition"] = dff["SaleCondition"].apply(lambda x: "Other" if x in ["Abnormal_Adjland", "Alloca_Family"] else x)
 dff["GarageType"] = dff["GarageType"].apply(lambda x: "Carport_None" if x in ["CarPort", "None"] else x)
 dff["GarageType"] = dff["GarageType"].apply(lambda x: "Basement_2Types" if x in ["Basment", "2Types"] else x)
 dff["LotConfig"] = dff["LotConfig"].apply(lambda x: "CulDSac_FR3" if x in ["CulDSac", "FR3"] else x)
 dff["RoofStyle"] = dff["RoofStyle"].apply(lambda x: "Other" if x not in ["Gable"] else x)
 dff["RoofMatl"] = dff["RoofMatl"].apply(lambda x: "Other" if x != "CompShg" else x)
 dff["MasVnrType"] = dff["MasVnrType"].apply(lambda x: "None_BrkCmn" if x in ["None", "BrkCmn"] else x)
 dff["Foundation"] = dff["Foundation"].apply(lambda x: "BrkTil_Stone" if x in ["BrkTil", "Stone"] else x)
 dff["Foundation"] = dff["Foundation"].apply(lambda x: "BrkTil_Stone_Slab" if x in ["BrkTil_Stone", "Slab"] else x)
 dff["Foundation"] = dff["Foundation"].apply(lambda x: "PConc_Wood" if x in ["PConc", "Wood"] else x)
 dff["Heating"] = dff["Heating"].apply(lambda x: "Other" if x != "GasA" else x )<feature_engineering>
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019 )
dff["FrontageRatio"] =(dff["LotFrontage"] / dff["LotArea"])
 dff["HQFloor"] = dff["1stFlrSF"] + dff["2ndFlrSF"]
dff["FloorAreaRatio"] = dff["GrLivArea"] / dff["LotArea"]
dff["TotalArea"] = dff["TotalBsmtSF"] + dff["GrLivArea"]
dff["TotalPorch"] = dff["WoodDeckSF"] + dff["OpenPorchSF"] + dff["EnclosedPorch"] + dff["3SsnPorch"] + dff["ScreenPorch"]
dff["WeightedBsmtFinSF1"] = dff["BsmtFinSF1"] * dff["BsmtFinType1"]
dff["WeightedBsmtFinSF2"] = dff["BsmtFinSF2"] * dff["BsmtFinType2"]
dff["WeightedTotalBasement"] = dff["WeightedBsmtFinSF1"] + dff["BsmtFinSF2"] * dff["BsmtFinType2"] + dff["BsmtUnfSF"]
dff["TotalFullBath"] = dff["BsmtFullBath"] + dff["FullBath"]
dff["TotalHalfBath"] = dff["BsmtHalfBath"] + dff["HalfBath"]
dff["TotalBsmtBath"] = dff["BsmtFullBath"] + 0.5 * dff["BsmtHalfBath"]
dff["TotalBath"] = dff["TotalFullBath"] + 0.5 *(dff["BsmtHalfBath"] + dff["HalfBath"])+ dff["BsmtFullBath"] + 0.5 * dff["BsmtHalfBath"]
dff["HasPool"] = dff["PoolArea"].apply(lambda x: 0 if x == 0 else 1)
 dff["Has2ndFlr"] = dff["2ndFlrSF"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasBsmt"] = dff["TotalBsmtSF"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasFireplace"] = dff["Fireplaces"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasGarage"] = dff["GarageArea"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasLowQual"] = dff["LowQualFinSF"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasPorch"] = dff["TotalPorch"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasMiscVal"] = dff["MiscVal"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasWoodDeck"] = dff["WoodDeckSF"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasOpenPorch"] = dff["OpenPorchSF"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasEnclosedPorch"] = dff["EnclosedPorch"].apply(lambda x: 0 if x == 0 else 1)
 dff["Has3SsnPorch"] = dff["3SsnPorch"].apply(lambda x: 0 if x == 0 else 1)
 dff["HasScreenPorch"] = dff["ScreenPorch"].apply(lambda x: 0 if x == 0 else 1)
 dff["TotalPorchType"] = dff["HasWoodDeck"] + dff["HasOpenPorch"] + dff["HasEnclosedPorch"] + dff["Has3SsnPorch"] + dff["HasScreenPorch"]
dff["TotalPorchType"] = dff["TotalPorchType"].apply(lambda x: 3 if x >=3 else x )<feature_engineering>
oof = np.zeros(len(train_df))
predictions = np.zeros(( len(test_df),nfold))
i = 1
for train_index, valid_index in skf.split(train_df, target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=target.iloc[train_index].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=target.iloc[valid_index].values,
feature_name=predictors,
free_raw_data = False 
)
 clf = lgb.train(param_lgb, xg_train, 10000, valid_sets = [xg_valid], verbose_eval=1000)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=clf.best_iteration)
 predictions[:,i-1] += clf.predict(test_df[predictors], num_iteration=clf.best_iteration)
 i = i + 1
print("
CV AUC: {:<0.6f}".format(metrics.roc_auc_score(target.values, oof)) )
dff["RestorationAge"] = dff["YearRemodAdd"] - dff["YearBuilt"]
dff["RestorationAge"] = dff["RestorationAge"].apply(lambda x: 0 if x < 0 else x)
 dff["HasRestoration"] = dff["RestorationAge"].apply(lambda x: 0 if x == 0 else 1)
 dff["YearAfterRestoration"] = dff["YrSold"] - dff["YearRemodAdd"]
dff["YearAfterRestoration"] = dff["YearAfterRestoration"].apply(lambda x: 0 if x < 0 else x)
 dff["BuildAge"] = dff["YrSold"] - dff["YearBuilt"]
dff["BuildAge"] = dff["BuildAge"].apply(lambda x: 0 if x < 0 else x)
 dff["IsNewHouse"] = dff["BuildAge"].apply(lambda x: 1 if x == 0 else 0)
 def year_map(year):
year = 1 if year <= 1895 else\ 
(2 if year <= 1916 else\ 
(3 if year <= 1919 else\ 
(4 if year <= 1929 else\ 
(5 if year <= 1941 else\ 
(6 if year <= 1945 else\ 
(7 if year <= 1964 else\ 
(8 if year <= 1980 else\ 
(9 if year <= 1991 else\ 
(10 if year < 2008 else 11)))))))))
 return year
dff["YearBuilt_bins"] = dff["YearBuilt"].apply(lambda year: year_map(year))
dff["YearRemodAdd_bins"] = dff["YearRemodAdd"].apply(lambda year: year_map(year))
dff["GarageYrBlt_bins"] = dff["GarageYrBlt"].apply(lambda year: year_map(year))
dff["YrSold"] = dff["YrSold"].astype(str)
 dff["MoSold"] = dff["MoSold"].astype(str)
 dff["Season"] = dff["MoSold"].apply(lambda x: "Winter" if x in ["12", "1", "2"] else\ 
("Spring" if x in ["3", "4", "5"] else\ 
("Summer" if x in ["6", "7", "8"] else "Fall")) )<feature_engineering>
print("Rank averaging on", nfold, "fold predictions")
 rank_predictions = np.zeros(( predictions.shape[0],1))
for i in range(nfold):
rank_predictions[:, 0] = np.add(rank_predictions[:, 0], rankdata(predictions[:, i].reshape(-1,1)) /rank_predictions.shape[0])
 rank_predictions /= nfold
dff["OverallValue"] = dff["OverallQual"] * dff["OverallCond"]
dff["ExterValue"] = dff["ExterQual"] * dff["ExterCond"]
dff["BsmtValue"] =(( dff["BsmtQual"] + dff["BsmtFinType1"] + dff["BsmtFinType2"])* dff["BsmtCond"])/ 2
dff["KitchenValue"] = dff["KitchenAbvGr"] * dff["KitchenQual"]
dff["FireplaceValue"] = dff["Fireplaces"] * dff["FireplaceQu"]
dff["GarageValue"] = dff["GarageQual"] * dff["GarageCond"]
dff["TotalValue"] = dff["OverallValue"] + dff["ExterValue"] + dff["BsmtValue"] + dff["KitchenValue"] + dff["FireplaceValue"] + dff["GarageValue"] +\
dff["HeatingQC"] + dff["Utilities"] + dff["Electrical"] - dff["Functional"] + dff["PoolQC"]
dff["TotalQual"] = dff["OverallQual"] + dff["ExterQual"] + dff["BsmtQual"] + dff["KitchenQual"] + dff["FireplaceQu"] + dff["GarageQual"] +\
dff["HeatingQC"] + dff["PoolQC"]
dff["TotalCond"] = dff["OverallCond"] + dff["ExterCond"] + dff["BsmtCond"] + dff["GarageCond"]
dff["TotalQualCond"] = dff["TotalQual"] + dff["TotalCond"]<feature_engineering>
sub_df = pd.DataFrame({"ID_code": ID_code.values})
 sub_df["target"] = rank_predictions
sub_df[:10]
<create_dataframe><EOS> 
sub_df.to_csv("Customer_Transaction_rank_predictions.csv", index=False )