Kaggle competition: Amazon.com - Employee Access Challenge
<SOS> metric: AUC Kaggle data source: amazon.com-employee-access-challenge<save_to_csv>
import numpy as np
import pandas as pd
import os
kaggle_data = test_data[['PassengerId', 'Predictions']].copy()
kaggle_data.rename(columns={'Predictions' : 'Survived'}, inplace=True)
 kaggle_data.sort_values(by=['PassengerId'] ).to_csv('kaggle_out_svc_names.csv', index=False )<set_options>
train, test = amazon()
target = "ACTION"
col4train = [x for x in train.columns if x not in [target, "ROLE_TITLE"]]
y = train[target].values
warnings.filterwarnings("ignore")
 with warnings.catch_warnings() :
warnings.filterwarnings("ignore",category=DeprecationWarning )<load_from_csv>
def get_model() :
params = {
"n_estimators":300,
"n_jobs": 3,
"random_state":5436,
}
return ExtraTreesClassifier(**params )
def import_data(data):
df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/{}.csv' 
.format(data))
return df
train = import_data('train')
 train = train.set_index('Id')
 test = import_data('test')
 ID = test[['Id']]
test = test.set_index('Id' )<feature_engineering>
class TargetEncoding(BaseEstimator, TransformerMixin):
def __init__(self, columns_names):
self.columns_names = columns_names
self.learned_values = {}
self.dataset_mean = np.nan
def fit(self, X, y, **fit_params):
X_ = X.copy()
self.learned_values = {}
X_["__target__"] = y
for c in [x for x in X_.columns if x in self.columns_names]:
self.learned_values[c] =(X_[[c,"__target__"]] 
.groupby(c)["__target__"].mean() 
.reset_index())
 self.dataset_mean = np.mean(y)
 return self
def transform(self, X, **fit_params):
transformed_X = X[self.columns_names].copy()
for c in transformed_X.columns:
transformed_X[c] =(transformed_X[[c]] 
.merge(self.learned_values[c], on = c, how = 'left')
)["__target__"]
transformed_X = transformed_X.fillna(self.dataset_mean)
 return transformed_X
def fit_transform(self, X, y, **fit_params):
self.fit(X,y)
 return self.transform(X )
corrmat = train.corr()
top_corr_features = corrmat.index[abs(corrmat["SalePrice"])>0.6]
for kol in top_corr_features:
z = np.abs(stats.zscore(train[kol]))
train['z'] = z
train = train.drop('z', axis = 1 )<filter>
skf = StratifiedKFold(n_splits=5, random_state = 5451, shuffle = True)
 te = TargetEncoding(columns_names=col4train)
 X_tr = te.fit_transform(train, y ).values
scores = []
tr_scores = []
for train_index, test_index in skf.split(train, y):
train_df, valid_df = X_tr[train_index], X_tr[test_index]
train_y, valid_y = y[train_index], y[test_index]
model = get_model()
model.fit(train_df,train_y)
 predictions = model.predict_proba(valid_df)[:,1]
scores.append(roc_auc_score(valid_y, predictions))
train_preds = model.predict_proba(train_df)[:,1]
tr_scores.append(roc_auc_score(train_y, train_preds))
print("Train AUC score: {:.4f} Valid AUC score: {:.4f}, STD: {:.4f}".format(
 np.mean(tr_scores), np.mean(scores), np.std(scores)
))
train = train[(train['GrLivArea']<4500)]<concatenate>
scores = []
tr_scores = []
for train_index, test_index in skf.split(train, y):
train_df = train.loc[train_index,col4train].reset_index(drop = True)
 valid_df = train.loc[test_index,col4train].reset_index(drop = True)
 train_y, valid_y = y[train_index], y[test_index]
te = TargetEncoding(columns_names=col4train)
 X_tr = te.fit_transform(train_df, train_y ).values
X_val = te.transform(valid_df ).values
model = get_model()
model.fit(X_tr,train_y)
 predictions = model.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(valid_y, predictions))
train_preds = model.predict_proba(X_tr)[:,1]
tr_scores.append(roc_auc_score(train_y, train_preds))
print("Train AUC score: {:.4f} Valid AUC score: {:.4f}, STD: {:.4f}".format(
 np.mean(tr_scores), np.mean(scores), np.std(scores)
))
df = pd.concat([train.drop(['SalePrice'], axis=1), test], axis=0 )<feature_engineering>
class TargetEncodingSmoothing(BaseEstimator, TransformerMixin):
def __init__(self, columns_names,k, f):
self.columns_names = columns_names
self.learned_values = {}
self.dataset_mean = np.nan
self.k = k
self.f = f
def smoothing_func(self, N):
return 1 /(1 + np.exp(-(N-self.k)/self.f))
def fit(self, X, y, **fit_params):
X_ = X.copy()
self.learned_values = {}
self.dataset_mean = np.mean(y)
 X_["__target__"] = y
for c in [x for x in X_.columns if x in self.columns_names]:
stats =(X_[[c,"__target__"]] 
.groupby(c)["__target__"].
 agg(['mean', 'size']))
stats["alpha"] = self.smoothing_func(stats["size"])
 stats["__target__"] =(stats["alpha"]*stats["mean"]
+(1-stats["alpha"])*self.dataset_mean)
 stats =(stats 
.drop([x for x in stats.columns if x not in ["__target__",c]], axis = 1)
.reset_index())
 self.learned_values[c] = stats
self.dataset_mean = np.mean(y)
 return self
def transform(self, X, **fit_params):
transformed_X = X[self.columns_names].copy()
for c in transformed_X.columns:
transformed_X[c] =(transformed_X[[c]] 
.merge(self.learned_values[c], on = c, how = 'left')
)["__target__"]
transformed_X = transformed_X.fillna(self.dataset_mean)
 return transformed_X
def fit_transform(self, X, y, **fit_params):
self.fit(X,y)
 return self.transform(X )
def null_view(df):
null_view = df[[col for col in df.columns if df[col].isna().sum() > 0]].isna().sum().sort_values(ascending = False)
 null_view = pd.DataFrame(null_view, columns=['n_nulls'])
 null_view['pct_null'] = null_view.n_nulls.apply(lambda x: str(round(( x/len(df)) *100, 1)) +'%')
 print(null_view.head(10))
null_view(test)
 print('')
 null_view(train )<data_type_conversions>
%matplotlib inline
x = np.linspace(0,100,100)
 plot = pd.DataFrame()
te = TargetEncodingSmoothing([], 1,1)
 plot["k=1|f=1"] = te.smoothing_func(x)
 te = TargetEncodingSmoothing([], 33,5)
 plot["k=33|f=5"] = te.smoothing_func(x)
 te = TargetEncodingSmoothing([], 66,15)
 plot["k=66|f=15"] = te.smoothing_func(x)
 plot.plot(figsize =(15,8))
df['MSSubClass'] = df['MSSubClass'].apply(str)
 df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str)
 commonNa = [
'MSZoning', 'Electrical',
'KitchenQual', 'Exterior1st',
'Exterior2nd', 'SaleType',
'LotFrontage', 'Functional'
]
toZero = [
'MasVnrArea', 'GarageYrBlt',
'BsmtHalfBath', 'BsmtFullBath',
'GarageArea', 'GarageCars',
'BsmtFinSF1', 'BsmtFinSF2',
'BsmtUnfSF', 'TotalBsmtSF'
]
toNoData = [
'PoolQC', 'MiscFeature',
'Fence','MasVnrType',
'FireplaceQu', 'GarageType',
'GarageFinish', 'GarageQual',
'GarageCond', 'BsmtQual',
'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2',
'Alley'
]
def fillNa_fe(df):
df['Functional'] = df['Functional'].fillna('Typ')
 for i in commonNa:
df[i] = df[i].fillna(df[i].mode() [0])
 for i in toNoData:
df[i] = df[i].fillna('None')
 for i in toZero:
df[i] = df[i].fillna(0)
 df.drop(['Utilities', 'PoolQC', 'MiscFeature'], axis=1, inplace=True)
 return df
df = fillNa_fe(df )<feature_engineering>
scores = []
tr_scores = []
for train_index, test_index in skf.split(train, y):
train_df = train.loc[train_index,col4train].reset_index(drop = True)
 valid_df = train.loc[test_index,col4train].reset_index(drop = True)
 train_y, valid_y = y[train_index], y[test_index]
te = TargetEncodingSmoothing(
 columns_names= col4train,
k = 3, f = 1.5 
)
 X_tr = te.fit_transform(train_df, train_y ).values
X_val = te.transform(valid_df ).values
model = get_model()
model.fit(X_tr,train_y)
 predictions = model.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(valid_y, predictions))
train_preds = model.predict_proba(X_tr)[:,1]
tr_scores.append(roc_auc_score(train_y, train_preds))
print("Train AUC score: {:.4f} Valid AUC score: {:.4f}, STD: {:.4f}".format(
 np.mean(tr_scores), np.mean(scores), np.std(scores)
))
def skew_(df):
colls = [col for col in df.columns if df[col].dtype in ['int64','float']]
skews_df = [col for col in df[colls].columns if abs(df[col].skew())>.7]
def skewfix(data, data2):
for i in data2:
data[i] = np.log1p(data[i])
 return data
return skewfix(df, skews_df)
 df = skew_(df )<feature_engineering>
def get_CV_target_encoding(data, y, encoder, cv = 5):
skfTE = StratifiedKFold(n_splits=cv, random_state = 545167, shuffle = True)
 result = []
for train_indexTE, test_indexTE in skfTE.split(data, y):
encoder.fit(data.iloc[train_indexTE,:].reset_index(drop = True), y[train_indexTE])
 tmp = encoder.transform(data.iloc[test_indexTE,:].reset_index(drop = True))
tmp["index"] = test_indexTE
result.append(tmp)
 result = pd.concat(result, ignore_index = True)
 result = result.sort_values('index' ).reset_index(drop = True ).drop('index', axis = 1)
 return result
def feat_eng(df):
df['totalSF'] = df['TotalBsmtSF'] \
+ df['1stFlrSF'] \
+ df['2ndFlrSF'] \
+ df['GrLivArea']
df['YrBltAndRemod']= df['YearBuilt'] + df['YearRemodAdd']
df['Porch_SF'] =(df['OpenPorchSF'] \
+ df['3SsnPorch'] \
+ df['EnclosedPorch'] \
+ df['ScreenPorch'] \
+ df['WoodDeckSF'])
 df['Total_Bathrooms'] =(df['FullBath'] \
+(0.5 * df['HalfBath'])\
+ df['BsmtFullBath'] \
+(0.5 * df['BsmtHalfBath']))
df['hpool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['h2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 return df
df = feat_eng(df )<categorify>
scores = []
tr_scores = []
for train_index, test_index in skf.split(train, y):
train_df = train.loc[train_index,col4train].reset_index(drop = True)
 valid_df = train.loc[test_index,col4train].reset_index(drop = True)
 train_y, valid_y = y[train_index], y[test_index]
te = TargetEncodingSmoothing(
 columns_names= col4train,
k = 3, f = 1.5 
)
 X_tr = get_CV_target_encoding(train_df, train_y, te, cv = 5)
 te.fit(train_df, train_y)
 X_val = te.transform(valid_df ).values
model = get_model()
model.fit(X_tr,train_y)
 predictions = model.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(valid_y, predictions))
train_preds = model.predict_proba(X_tr)[:,1]
tr_scores.append(roc_auc_score(train_y, train_preds))
print("Train AUC score: {:.4f} Valid AUC score: {:.4f}, STD: {:.4f}".format(
 np.mean(tr_scores), np.mean(scores), np.std(scores)
))
encoder = LabelEncoder()
sc = StandardScaler()
def encode(df):
cat_df = [col for col in df.columns if df[col].dtype not in ['int','float']]
for col in cat_df:
df[col] = encoder.fit_transform(df[col])
 df_ = sc.fit_transform(df)
 df = pd.DataFrame(data=df_, columns = df.columns)
 return df

 df = pd.get_dummies(df)
<split>
class TargetEncodingExpandingMean(BaseEstimator, TransformerMixin):
def __init__(self, columns_names):
self.columns_names = columns_names
self.learned_values = {}
self.dataset_mean = np.nan
def fit(self, X, y, **fit_params):
X_ = X.copy()
self.learned_values = {}
self.dataset_mean = np.mean(y)
 X_["__target__"] = y
for c in [x for x in X_.columns if x in self.columns_names]:
stats =(X_[[c,"__target__"]] 
.groupby(c)["__target__"] 
.agg(['mean', 'size']))
stats["__target__"] = stats["mean"]
stats =(stats 
.drop([x for x in stats.columns if x not in ["__target__",c]], axis = 1)
.reset_index())
 self.learned_values[c] = stats
return self
def transform(self, X, **fit_params):
transformed_X = X[self.columns_names].copy()
for c in transformed_X.columns:
transformed_X[c] =(transformed_X[[c]] 
.merge(self.learned_values[c], on = c, how = 'left')
)["__target__"]
transformed_X = transformed_X.fillna(self.dataset_mean)
 return transformed_X
def fit_transform(self, X, y, **fit_params):
self.fit(X,y)
 X_ = X[self.columns_names].copy().reset_index(drop = True)
 X_["__target__"] = y
X_["index"] = X_.index
X_transformed = pd.DataFrame()
for c in self.columns_names:
X_shuffled = X_[[c,"__target__", "index"]].copy()
X_shuffled = X_shuffled.sample(n = len(X_shuffled),replace=False)
 X_shuffled["cnt"] = 1
X_shuffled["cumsum"] =(X_shuffled 
.groupby(c,sort=False)['__target__'] 
.apply(lambda x : x.shift().cumsum()))
X_shuffled["cumcnt"] =(X_shuffled 
.groupby(c,sort=False)['cnt'] 
.apply(lambda x : x.shift().cumsum()))
X_shuffled["encoded"] = X_shuffled["cumsum"] / X_shuffled["cumcnt"]
X_shuffled["encoded"] = X_shuffled["encoded"].fillna(self.dataset_mean)
 X_transformed[c] = X_shuffled.sort_values("index")["encoded"].values
return X_transformed
train = df.iloc[:train.shape[0],:]
test = df.iloc[train.shape[0]:,:]<split>
scores = []
tr_scores = []
for train_index, test_index in skf.split(train, y):
train_df = train.loc[train_index,col4train].reset_index(drop = True)
 valid_df = train.loc[test_index,col4train].reset_index(drop = True)
 train_y, valid_y = y[train_index], y[test_index]
te = TargetEncodingExpandingMean(columns_names=col4train)
 X_tr = te.fit_transform(train_df, train_y)
 X_val = te.transform(valid_df ).values
model = get_model()
model.fit(X_tr,train_y)
 predictions = model.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(valid_y, predictions))
train_preds = model.predict_proba(X_tr)[:,1]
tr_scores.append(roc_auc_score(train_y, train_preds))
print("Train AUC score: {:.4f} Valid AUC score: {:.4f}, STD: {:.4f}".format(
 np.mean(tr_scores), np.mean(scores), np.std(scores)
))
X = train.copy()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42 )<import_modules>
train[col4train] = train[col4train].values.astype(str)
 test[col4train] = test[col4train].values.astype(str)
 new_col4train = col4train
for c1,c2 in combinations(col4train, 2):
name = "{}_{}".format(c1,c2)
 new_col4train.append(name)
 train[name] = train[c1] + "_" + train[c2]
test[name] = test[c1] + "_" + test[c2]
import datetime
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV<init_hyperparams>
train[new_col4train].apply(lambda x: len(x.unique()))
print('
{}\tLightBGM' 
.format(datetime.datetime.now().strftime('%H:%M')))
 parameters = {
'objective':['regression'],
'num_leaves':[5],
'learning_rate':[0.05],
'n_estimators':[720],
'max_bin':[55],
'max_depth':[2, 3],
'bagging_fraction':[.5,.8],
'bagging_freq':[5],
'bagging_seed':[9],
'feature_fraction':[0.2319],
'feature_fraction_seed':[9]
}
light = LGBMRegressor()
clf = GridSearchCV(light, parameters, verbose=0, iid=False)
 clf.fit(X_train, y_train)
 lightgbm = LGBMRegressor(**clf.best_params_)
 print('
Regressor:
', lightgbm, '
')
 print('{}\tDone! 
.format(datetime.datetime.now().strftime('%H:%M')) )<choose_model_class>
scores = []
tr_scores = []
for train_index, test_index in skf.split(train, y):
train_df = train.loc[train_index,new_col4train].reset_index(drop = True)
 valid_df = train.loc[test_index,new_col4train].reset_index(drop = True)
 train_y, valid_y = y[train_index], y[test_index]
te = TargetEncodingExpandingMean(columns_names=new_col4train)
 X_tr = te.fit_transform(train_df, train_y)
 X_val = te.transform(valid_df)
 te2 = TargetEncodingSmoothing(
 columns_names= new_col4train,
k = 3, f = 1.5, 
)
 X_tr2 = get_CV_target_encoding(train_df, train_y, te2, cv = 5)
 te2.fit(train_df, train_y)
 X_val2 = te2.transform(valid_df)
 X_tr = pd.concat([X_tr, X_tr2], axis = 1)
 X_val = pd.concat([X_val, X_val2], axis = 1)
 model = get_model()
model.fit(X_tr,train_y)
 predictions = model.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(valid_y, predictions))
train_preds = model.predict_proba(X_tr)[:,1]
tr_scores.append(roc_auc_score(train_y, train_preds))
print("Train AUC score: {:.4f} Valid AUC score: {:.4f}, STD: {:.4f}".format(
 np.mean(tr_scores), np.mean(scores), np.std(scores)
))
print('
{}\tXGBoost' 
.format(datetime.datetime.now().strftime('%H:%M')))
 params = {
'colsample_bytree': [0.4],
'gamma': [0.0],
'learning_rate': [0.01],
'max_depth': [3],
'min_child_weight': [2],
'n_estimators': [3460],
'seed': [36],
'subsample': [0.2],
'objective':['reg:squarederror'],
'reg_alpha':[0.00006],
'cale_pos_weight':[1],
'verbosity':[0]
}
gbm = XGBRegressor()
clf = GridSearchCV(gbm, params, verbose=0, iid=False)
 clf.fit(X_train,y_train)
 XGBoost = XGBRegressor(**clf.best_params_)
 print('
Regressor:
', XGBoost, '
')
 print('{}\tDone! 
.format(datetime.datetime.now().strftime('%H:%M')) )<train_on_grid>
te = TargetEncodingExpandingMean(columns_names=new_col4train)
 X_tr = te.fit_transform(train[new_col4train], y)
 X_val = te.transform(test[new_col4train])
 te2 = TargetEncodingSmoothing(
 columns_names= new_col4train,
k = 3, f = 1.5, 
)
 X_tr2 = get_CV_target_encoding(train[new_col4train], y, te2, cv = 5)
 te2.fit(train[new_col4train], y)
 X_val2 = te2.transform(test[new_col4train])
 X = pd.concat([X_tr, X_tr2], axis = 1)
 X_te = pd.concat([X_val, X_val2], axis = 1)
 model = get_model()
model.fit(X,y)
 predictions = model.predict_proba(X_te)[:,1]
submit = pd.DataFrame()
submit["Id"] = test["id"]
submit["ACTION"] = predictions
submit.to_csv("submission.csv", index = False )