Kaggle competition: House Prices - Advanced Regression Techniques
print("F1 Score = " + str(f1_score(df_train['target'], train_pred_int)) )<save_to_csv>
warnings.filterwarnings('ignore')
 sns.set(rc={'figure.figsize':(15,12)})
 %matplotlib inline
df_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 df_submission['target'] = test_pred_int
df_submission.to_csv("submission.csv", index=False, header=True )<set_options>
train_df = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_df = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
%matplotlib inline<load_from_csv>
train_ID = train_df['Id']
test_ID = test_df['Id']
train_df.drop("Id", axis = 1, inplace = True)
 test_df.drop("Id", axis = 1, inplace = True)
 train_df = train_df.drop(train_df[(train_df['GrLivArea']>4000)&(train_df['SalePrice']<300000)].index )
df_train = pd.read_csv(".. /input/santander-customer-satisfaction/train.csv")
 df_test = pd.read_csv(".. /input/santander-customer-satisfaction/test.csv" )<count_missing_values>
ntrain = train_df.shape[0]
ntest = test_df.shape[0]
y_train = train_df.SalePrice.values
comp_data = pd.concat(( train_df, test_df)).reset_index(drop=True)
 comp_data.drop(['SalePrice'], axis=1, inplace=True)
 print("Comp_data size is : {}".format(comp_data.shape))
df_train.isnull().sum().sum() , df_test.isnull().sum().sum()<count_values>
missing_val = comp_data.isnull().sum().sort_values(ascending=False)
 missing_val_df = pd.DataFrame({'Feature':missing_val.index, 'Count':missing_val.values})
 missing_val_df = missing_val_df.drop(missing_val_df[missing_val_df.Count == 0].index)
 missing_val_df
rate_insatisfied = df_train.TARGET.value_counts() [1] / df_train.TARGET.value_counts() [0]
rate_insatisfied * 100<split>
comp_data["LotFrontage"] = comp_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
comp_data[col] = comp_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
comp_data[col] = comp_data[col].fillna(0)
 comp_data['MSZoning'] = comp_data['MSZoning'].fillna(comp_data['MSZoning'].mode() [0])
 comp_data["MasVnrArea"] = comp_data["MasVnrArea"].fillna(0)
 comp_data['Electrical'] = comp_data['Electrical'].fillna(comp_data['Electrical'].mode() [0])
 comp_data['SaleType'] = comp_data['SaleType'].fillna(comp_data['SaleType'].mode() [0])
 comp_data['KitchenQual'] = comp_data['KitchenQual'].fillna(comp_data['KitchenQual'].mode() [0])
 comp_data['Exterior1st'] = comp_data['Exterior1st'].fillna(comp_data['Exterior1st'].mode() [0])
 comp_data['Exterior2nd'] = comp_data['Exterior2nd'].fillna(comp_data['Exterior2nd'].mode() [0] )
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('TARGET', axis = 1), df_train.TARGET,
train_size = 0.8, stratify = df_train.TARGET,
random_state = 42)
 X_train.shape, y_train.shape[0], X_test.shape, y_test.shape[0]<create_dataframe>
for col in('PoolQC', 'MiscFeature', 'Alley'):
comp_data[col] = comp_data[col].fillna('None')
 comp_data["MasVnrType"] = comp_data["MasVnrType"].fillna("None")
 comp_data["Fence"] = comp_data["Fence"].fillna("None")
 comp_data["FireplaceQu"] = comp_data["FireplaceQu"].fillna("None")
 for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
comp_data[col] = comp_data[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
comp_data[col] = comp_data[col].fillna('None')
 comp_data['MSSubClass'] = comp_data['MSSubClass'].fillna("None")
 comp_data['SaleType'] = comp_data['SaleType'].fillna(comp_data['SaleType'].mode() [0])
 comp_data = comp_data.drop(['Utilities'], axis=1)
 comp_data['OverallCond'] = comp_data['OverallCond'].astype(str)
 comp_data["Functional"] = comp_data["Functional"].fillna("Typ")
X_train_clean = X_train.copy()
X_test_clean = X_test.copy()<features_selection>
comp_data['MSSubClass'] = comp_data['MSSubClass'].apply(str)
 comp_data['YrSold'] = comp_data['YrSold'].astype(str)
 comp_data['MoSold'] = comp_data['MoSold'].astype(str)
 columns =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for col in columns:
labl = LabelEncoder()
labl.fit(list(comp_data[col].values))
comp_data[col] = labl.transform(list(comp_data[col].values))
print('Shape all_data: {}'.format(comp_data.shape))
comp_data['TotalSF'] = comp_data['TotalBsmtSF'] + comp_data['1stFlrSF'] + comp_data['2ndFlrSF']
selector = VarianceThreshold(threshold = 0.01)
 selector.fit(X_train_clean)
 mask_clean = selector.get_support()
X_train_clean = X_train_clean[X_train_clean.columns[mask_clean]]<define_variables>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
comp_data[feat] = boxcox1p(comp_data[feat], lam )
( len(df_train.columns)- 1)- X_train_clean.shape[1]<drop_column>
comp_data = pd.get_dummies(comp_data)
 print(comp_data.shape )
remove = []
cols = X_train_clean.columns
for i in range(len(cols)-1):
column = X_train_clean[cols[i]].values
for j in range(i+1,len(cols)) :
if np.array_equal(column, X_train_clean[cols[j]].values):
remove.append(cols[j])
 X_train_clean.drop(remove, axis = 1, inplace=True )<import_modules>
train_df = comp_data[:ntrain]
test_df = comp_data[ntrain:]
from sklearn.feature_selection import SelectKBest, f_classif, mutual_info_classif
from sklearn.metrics import roc_auc_score as auc
from sklearn.model_selection import cross_val_score
import xgboost as xgb<train_on_grid>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train_df.values)
 rmse= np.sqrt(-cross_val_score(model, train_df.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
K_vs_score_fc = []
start = time.time()
for k in range(2, 247, 2):
start = time.time()
selector_fc = SelectKBest(score_func = f_classif, k = k)
 X_train_selected_fc = selector_fc.fit_transform(X_train_clean, y_train)
 clf = xgb.XGBClassifier(seed=42)
 auc_fc = cross_val_score(clf, X_train_selected_fc, y_train, cv = 10, scoring = 'roc_auc')
 K_vs_score_fc.append(auc_fc.mean())
 end = time.time()
print("k = {} - auc_fc = {} - Time = {}s".format(k, auc_fc.mean() , end-start))
end = time.time()
print(end - start)
<train_on_grid>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
K_vs_score_mic = []
for k in range(2, 247, 2):
start = time.time()
selector_mic = SelectKBest(score_func = mutual_info_classif, k = k)
 X_train_selected_mic = selector_mic.fit_transform(X_train_clean, y_train)
 clf = xgb.XGBClassifier(seed=42)
 auc_mic = cross_val_score(clf, X_train_selected_mic, y_train, cv = 10, scoring = 'roc_auc')
 K_vs_score_mic.append(auc_mic.mean())
 end = time.time()
print("k = {} - auc_mic = {} - Time = {}s".format(k, auc_mic.mean() , end-start)) 
<train_on_grid>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
selector_fc = SelectKBest(score_func = f_classif, k = 96)
 selector_fc.fit(X_train_clean, y_train)
 mask_selected = selector_fc.get_support()
selected_col = X_train_clean.columns[mask_selected]
selected_col<filter>
model_xgb.fit(train_df, y_train)
 xgb_train_pred = model_xgb.predict(train_df)
 xgb_pred = np.expm1(model_xgb.predict(test_df))
print(rmsle(y_train, xgb_train_pred))
X_train_selected = X_train[selected_col]
X_test_selected = X_test[selected_col]<import_modules>
model_lgb.fit(train_df, y_train)
 lgb_train_pred = model_lgb.predict(train_df)
 lgb_pred = np.expm1(model_lgb.predict(test_df.values))
print(rmsle(y_train, lgb_train_pred))
from skopt import forest_minimize<choose_model_class>
ensemble = xgb_pred*0.5 + lgb_pred*0.5
<choose_model_class><EOS> 
submission = pd.DataFrame()
submission['Id'] = test_ID
submission['SalePrice'] = ensemble
print("Creating Submission File")
 submission.to_csv("submission.csv", index=False )