Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<feature_engineering>
%matplotlib inline
dff2["LotShape"] = dff2["LotShape"].apply(lambda x: 1 if x in [1, 2] else(2 if x == 3 else 3))
dff2["LandSlope"] = dff2["LandSlope"].apply(lambda x: 1 if x in [1, 2] else(2 if x == 3 else 3))
dff2["OverallCond"] = dff2["OverallCond"].apply(lambda x: 1 if x in [1, 2, 3] else x-1)
 dff2["OverallQual"] = dff2["OverallQual"].apply(lambda x: 1 if x in [1, 2] else x-1)
 dff2["ExterCond"] = dff2["ExterCond"].apply(lambda x: 1 if x in [1, 2] else(2 if x == 3 else 3))
dff2["BsmtQual"] = dff2["BsmtQual"].apply(lambda x: 0 if x in [0, 1, 2] else(1 if x == 3 else(2 if x == 4 else 3)))
 dff2["BsmtCond"] = dff2["BsmtCond"].apply(lambda x: 0 if x in [0, 1, 2] else(1 if x == 3 else 2))
dff2["BsmtFinType1"] = dff2["BsmtFinType1"].apply(lambda x: 1 if x in [1, 2, 3, 4, 5] else(2 if x == 6 else x))
dff2["BsmtFinType2"] = dff2["BsmtFinType2"].apply(lambda x: 1 if x in [1, 2, 3, 4, 5] else(2 if x == 6 else x))
dff2["HeatingQC"] = dff2["HeatingQC"].apply(lambda x: 1 if x in [1, 2] else(2 if x in [3, 4] else 3))
dff2["Electrical"] = dff2["Electrical"].apply(lambda x: 1 if x in [1, 2] else x-3)
 dff2["BsmtFullBath"] = dff2["BsmtFullBath"].apply(lambda x: 2 if x >= 2 else x)
 dff2["FullBath"] = dff2["FullBath"].apply(lambda x: 1 if x <= 1 else(3 if x >= 3 else x))
dff2["HalfBath"] = dff2["HalfBath"].apply(lambda x: 1 if x >= 1 else 0)
 dff2["BedroomAbvGr"] = dff2["BedroomAbvGr"].apply(lambda x: 1 if x <=1 else(5 if x >= 5 else x))
dff2["KitchenAbvGr"] = dff2["KitchenAbvGr"].apply(lambda x: 1 if x <= 1 else(2 if x >= 2 else x))
dff2["TotRmsAbvGrd"] = dff2["TotRmsAbvGrd"].apply(lambda x: 3 if x <= 4 else(10 if x >= 11 else x-1))
dff2["Functional"] = dff2["Functional"].apply(lambda x: 1 if x == 1 else 2)
 dff2["Fireplaces"] = dff2["Fireplaces"].apply(lambda x: 2 if x >= 2 else x)
 dff2["GarageCars"] = dff2["GarageCars"].apply(lambda x: 3 if x >= 3 else x)
 dff2["GarageQual"] = dff2["GarageQual"].apply(lambda x: 1 if x <= 2 else(2 if x == 3 else 3))
dff2["GarageCond"] = dff2["GarageCond"].apply(lambda x: 1 if x <= 2 else 2)
 dff2["Fence"] = dff2["Fence"].apply(lambda x: 1 if x in [1, 3] else x )<create_dataframe>
Kaggle=1
if Kaggle==0:
train=pd.read_csv("train.csv")
 test=pd.read_csv("test.csv")
 sample_sub=pd.read_csv("sample_submission.csv")
 else:
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 sample_sub=pd.read_csv(".. /input/sample_submission.csv" )
dff3 = dff2.copy()
dff3<count_values>
train['target'].value_counts()
for col in cat_cols2:
print(col, dff3[col].value_counts().size )<feature_engineering>
from sklearn.model_selection import train_test_split,KFold, cross_val_score, GridSearchCV,StratifiedKFold,StratifiedShuffleSplit
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, roc_auc_score
import eli5
from eli5.sklearn import PermutationImportance
from sklearn.preprocessing import StandardScaler
import xgboost as xgb
import lightgbm as lgb
from tqdm import tqdm_notebook
import feather
from bayes_opt import BayesianOptimization
target_encoding = ["MSSubClass", "Neighborhood", "Exterior1st", "Exterior2nd", "Condition1", "Condition2", "HouseStyle"]
for col in target_encoding:
feature_name = col + "Rank"
dff3.loc[:, feature_name] = dff3[col].map(dff3.groupby(col ).SalePrice.median())
 dff3.loc[:, feature_name] = dff3.loc[:, feature_name].rank(method = "dense" )<feature_engineering>
train_model=train.drop('ID_code',axis=1)
 test_model=test.drop('ID_code',axis=1 )
dff3["Exterior"] = np.where(( dff3["Exterior1st"] != dff3["Exterior2nd"]), "Mixed", dff3["Exterior1st"])
 dff3["No2ndExt"] = dff3["Exterior"].apply(lambda x: 0 if x == "Mixed" else 1 )<drop_column>
X=train_model.drop('target',axis=1)
 Y=train_model['target']
drop_cols = ["MSSubClass", "Neighborhood", "Condition1", "Condition2", "Exterior1st", "Exterior2nd", "PoolArea", "PoolQC",
"WoodDeckSF", "OpenPorchSF", "EnclosedPorch", "3SsnPorch", "ScreenPorch", "LowQualFinSF", "MiscVal", "2ndFlrSF",
"HouseStyle","YrSold", "MoSold", "YearBuilt", "YearRemodAdd", "GarageYrBlt", "Exterior", "Utilities", "Street"]
dff3.drop(drop_cols, axis = 1, inplace = True )<categorify>
train_x,valid_x,train_y,valid_y=train_test_split(X,Y,test_size=0.2,stratify=Y,random_state=100 )
num_, cat_, ord_ = col_types(dff3)
 def prep_data(df, cat_cols, target):
dummies = pd.get_dummies(df[cat_cols], drop_first = True)
 data = pd.concat([df, dummies], axis = 1 ).drop(cat_cols, axis = 1)
 train = data[data[target].notnull() ]
test = data[data[target].isnull() ]
return train, test
train, test = prep_data(dff3, cat_, "SalePrice")
 target = "SalePrice"
predictors = [x for x in train.columns if x not in ["Id", "SalePrice"]]<drop_column>
print(f'Training has {train_x.shape[0]} rows and {train_x.shape[1]} columns')
 print(f'Validation has {valid_x.shape[0]} rows and {valid_x.shape[1]} columns' )
high_skew = skewed[skewed.TrainSkewness_AfterYeoJohnson.abs() > 1].Feature.tolist()
print(high_skew)
 train.drop(high_skew, axis = 1, inplace = True)
 test.drop(high_skew, axis = 1, inplace = True )<concatenate>
train_y.value_counts()
train[target] = np.log1p(train[target])
 feature_distribution(train, target, test = False)
 train[target].skew()<define_variables>
valid_y.value_counts()
target = "SalePrice"
predictors = [x for x in train.columns if x not in ["Id", "SalePrice"]]<normalization>
feature_name=[f for f in train_x.columns if f not in ['target']]
mean_auc=0.0
N_SPLITS=5
scaler = RobustScaler()
train[predictors] = scaler.fit_transform(train[predictors])
 test[predictors] = scaler.transform(test[predictors] )<train_model>
X_train = train[predictors]
y_train = train[target]
X_test = test[predictors]
print(X_train.shape)
 print(X_test.shape )<normalization>
rf_oof_preds=np.zeros(train_x.shape[0])
 rf_sub_preds=np.zeros(test_model.shape[0] )
selector = VarianceThreshold(0.01)
 selector.fit(X_train)
 X_train = X_train[X_train.columns[selector.get_support() ]]
selector.transform(X_test)
 X_test = X_test[X_test.columns[selector.get_support() ]]
print(X_train.shape)
 print(X_test.shape)
 X_test<define_variables>
folds=StratifiedShuffleSplit(n_splits=5,random_state=100 )
test_id = test[["Id"]]<compute_train_metric>
auc_score=[]
importance=pd.DataFrame()
%time
for n_folds,(train_idx,valid_idx)in enumerate(folds.split(train_x,train_y)) :
x_train,x_valid=train_x.iloc[train_idx],train_x.iloc[valid_idx]
y_train,y_valid=train_y.iloc[train_idx],train_y.iloc[valid_idx]
clf=RandomForestClassifier(n_estimators=720 ,max_depth= 50,min_samples_leaf=9 ,max_features=2 ,n_jobs=-1,random_state=100,verbose=True)
 clf.fit(x_train,y_train)
 y_preds_proba=clf.predict_proba(x_valid)
 rf_oof_preds[valid_idx]=y_preds_proba[:,1]
rf_sub_preds=clf.predict_proba(test_model[feature_name])[:,1]/folds.n_splits
auc_score.append(roc_auc_score(y_valid,rf_oof_preds[valid_idx]))
print("
{} fold ROC AUC Score is : {}".format(n_folds+1,roc_auc_score(y_valid,rf_oof_preds[valid_idx])))
 importance['feature']=feature_name
importance['gini']=clf.feature_importances_
importance['fold']=n_folds+1
print("
Average ROC Score is {}",np.mean(auc_score))
kf = KFold(n_splits = 10, shuffle = True, random_state = 42)
 def rmse_cv(model, X = X_train, y = y_train):
return np.sqrt(-cross_val_score(model, X, y, scoring = 'neg_mean_squared_error', cv = kf)).mean()<choose_model_class>
lgb_oof_preds=np.zeros(train_x.shape[0])
 lgb_sub_preds=np.zeros(test_model.shape[0])
 auc_valid=[]
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
lgb_model = lgb.LGBMRegressor(colsample_bytree=0.25, learning_rate=0.01,
max_depth=13, min_child_samples=7, n_estimators=10000,
num_leaves=20, objective='regression', random_state=42,
subsample=0.9330025956033094, subsample_freq=1)
 xgb_model = xgb.XGBRegressor(colsample_bytree=0.25, gamma=0.0, learning_rate=0.01, max_depth=3,
n_estimators=15000, n_jobs=-1, random_state=42,
reg_alpha=0.24206673672530965, reg_lambda=0.40464485640717085, subsample=1.0)
 gbr_model = GradientBoostingRegressor(alpha=0.8979588317644014,
learning_rate=0.01, loss='huber',
max_depth=13, max_features=0.1, min_samples_split=109,
n_estimators=10000, n_iter_no_change=100, random_state=42)
 svr_model = SVR(C=0.7682824405204463, coef0=0.0001, degree=2, epsilon=0.0001, gamma=0.0042151786393578635, max_iter=10000)
 lasso_model = Lasso(alpha=0.00012609086150256233, max_iter=5000, random_state=42)
 ridge_model = Ridge(alpha=2.651347536470113, max_iter=5000, random_state=42)
 enet_model = ElasticNet(alpha=0.0002286518512853544, l1_ratio=0.6510386358323069, max_iter=5000, random_state=42 )<create_dataframe>
importances=pd.DataFrame()
for fold_idx,(train_ids, valid_ids)in enumerate(folds.split(train_x,train_y)) :
trn_data = lgb.Dataset(train_x.iloc[train_ids],label=train_y.iloc[train_ids])
 val_data = lgb.Dataset(train_x.iloc[valid_ids],label=train_y.iloc[valid_ids])
 num_rounds=10000
clf = lgb.train(param, trn_data, num_rounds, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 lgb_oof_preds[valid_ids] = clf.predict(train_x.iloc[valid_ids],num_iteration=clf.best_iteration)
 imp_df = pd.DataFrame()
imp_df['feature'] = feature_name
imp_df['gain'] = clf.feature_importance()
imp_df['fold'] = fold_idx + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 lgb_sub_preds += clf.predict(test_model[feature_name],num_iteration=clf.best_iteration)/ folds.n_splits
gc.collect()
print("Mean AUC: %.5f" %(roc_auc_score(train_y,lgb_oof_preds)) )
<train_model><EOS> 
sample_submission = pd.DataFrame({"ID_code":test["ID_code"].values})
 sample_submission["target"] =(0.4*rf_sub_preds)+(0.6*lgb_sub_preds)
 sample_submission.to_csv("blend_submission.csv", index=False )