Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
%matplotlib inline
g_boost = GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse',
init=None, learning_rate=0.01, loss='huber',
max_depth=3, max_features=18, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=3, min_samples_split=5,
min_weight_fraction_leaf=0.0, n_estimators=3300,
n_iter_no_change=None,random_state=None, subsample=1.0, tol=0.0001,
validation_fraction=0.1, verbose=0, warm_start=False )<define_search_model>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 print(train.shape,test.shape )
<choose_model_class>
na_columns = [col for col in train.columns if train[col].isna().sum() >=1]
for col in na_columns:
print(col,np.round(train[col].isna().mean() ,4))
xg_boost = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1, missing=None, n_estimators=3300,
n_jobs=1, nthread=None, objective='reg:linear', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=0.3, verbosity=1 )<choose_model_class>
numerical_col = [col for col in train.columns if train[col].dtypes !='O']
print(numerical_col)
 print(len(numerical_col))
<choose_model_class>
descrete_col = [col for col in numerical_col if len(train[col].unique())<25 and col not in year_col]
len(descrete_col )
lgbm = LGBMRegressor(bagging_fraction=0.6, bagging_freq=9, bagging_seed=14,
boosting_type='gbdt', class_weight=None, colsample_bytree=0,
feature_fraction=0.1, feature_fraction_seed=1,
importance_type='split', learning_rate=0.01, max_bin=65,
max_depth=4, min_child_samples=20, min_child_weight=0.001,
min_data_in_leaf=5, min_split_gain=0.0, min_sum_hessian_in_leaf=5,
n_estimators=3300, n_jobs=-1, num_leaves=7,
objective='regression', random_state=None, reg_alpha=0.2,
reg_lambda=0.1, silent=True, subsample=1.0,
subsample_for_bin=200000, subsample_freq=0 )<train_on_grid>
conti_col = [col for col in numerical_col if col not in descrete_col+year_col+['Id']]
print(len(conti_col),'continuous features')
 print(conti_col )
stacking = StackingRegressor(regressors=(elastic, g_boost, k_ridge),
meta_regressor = lasso)
 param_grid = {}
stack = GridSearchCV(stacking,
param_grid = param_grid,
cv = 10,
scoring = "neg_mean_squared_error",
n_jobs = 5,
verbose = 1)
 stack.fit(X_train,Y_train)
 s_best = stack.best_estimator_
stack.best_score_<find_best_params>
cat_col = [col for col in train.columns if train[col].dtypes == 'O']
print(cat_col)
 print('We have {} categorical features'.format(len(cat_col)) )
stacking = s_best<choose_model_class>
for col in cat_col:
print(col,': {}'.format(train[col].nunique()))
voting = VotingRegressor(estimators=[('xgboost', xg_boost), 
('lgbm', lgbm), 
('stacking', stacking)])
 v_param_grid = {}
gsV = GridSearchCV(voting,
param_grid = v_param_grid,
cv = 10,
scoring = "neg_mean_squared_error",
n_jobs = 5,
verbose = 1)
 gsV.fit(X_train,Y_train)
 v_best = gsV.best_estimator_
gsV.best_score_<find_best_params>
y = train['SalePrice'].reset_index(drop=True)
 previous_train = train.copy()
voting = v_best<predict_on_test>
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True )
y_vote = np.expm1(voting.predict(X_test))
lasso.fit(X_train,Y_train)
 y_lasso = np.expm1(lasso.predict(X_test))<define_variables>
nan_feature = [col for col in all_data.columns if all_data[col].isnull().sum() >=1 and all_data[col].dtypes == 'O']
print(len(nan_feature))
for feture in nan_feature:
print('{} has {} % missing values'.format(feture,np.round(all_data[feture].isnull().mean() ,4)) )
y_pred =(0.6*y_vote + 0.4*y_lasso )<save_to_csv>
def replace_missing(data,missing_feature):
df = data.copy()
df[missing_feature] = df[missing_feature].fillna('missing')
 return df
all_data = replace_missing(all_data,nan_feature)
 all_data[nan_feature].isnull().sum()
submission = pd.DataFrame(test_ids.values, columns = ["Id"])
 submission['SalePrice'] = list(y_pred)
 submission.to_csv('submission.csv',index=False )<set_options>
num_na_col = [col for col in all_data.columns if all_data[col].isnull().sum() >=1 and all_data[col].dtypes != 'O']
for col in num_na_col:
print('{} has {}% null values'.format(col,np.round(all_data[col].isnull().mean() ,4)) )
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42<load_from_csv>
for col in num_na_col:
median_value = all_data[col].median()
all_data[col+'NaN'] = np.where(all_data[col].isnull() ,1,0)
 all_data[col].fillna(median_value,inplace = True)
 all_data[num_na_col].isnull().sum()
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['Survived'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )<count_missing_values>
for col in ['YearBuilt','YearRemodAdd','GarageYrBlt']:
all_data[col] = all_data['YrSold']-all_data[col]
all_data[year_col].head()
def display_missing(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in dfs:
print('{}'.format(df.name))
display_missing(df )<feature_engineering>
num_col = ['LotFrontage', 'LotArea', '1stFlrSF', 'GrLivArea']
for col in num_col:
all_data[col] = np.log1p(all_data[col] )
age_by_pclass_sex = df_all.groupby(['Sex', 'Pclass'] ).median() ['Age']
for pclass in range(1, 4):
for sex in ['female', 'male']:
print('Median age of Pclass {} {}s: {}'.format(pclass, sex, age_by_pclass_sex[sex][pclass]))
print('Median age of all passengers: {}'.format(df_all['Age'].median()))
df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))<filter>
cat_col = [col for col in all_data.columns if all_data[col].dtypes == 'O']
for col in cat_col:
temp = all_data[col].value_counts() /len(all_data)
 temp_df = temp[temp>0.01].index
all_data[col] = np.where(all_data[col].isin(temp_df),all_data[col],'RareVar' )
df_all[df_all['Embarked'].isnull() ]<data_type_conversions>
cat_train = all_data.iloc[:len(y), :]
cat_test = all_data.iloc[len(y):, :]
df_all['Embarked'] = df_all['Embarked'].fillna('S' )<categorify>
all1 = pd.get_dummies(all_data,drop_first=True )
med_fare = df_all.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med_fare )<feature_engineering>
x_train = all1.iloc[:len(y), :]
x_test = all1.iloc[len(y):, :]
idx = df_all[df_all['Deck'] == 'T'].index
df_all.loc[idx, 'Deck'] = 'A'<groupby>
x_train.drop('Id',axis = 1,inplace = True)
 x_test.drop('Id',axis = 1,inplace = True)
 cat_train.drop('Id',axis = 1,inplace = True)
 cat_test.drop('Id',axis = 1,inplace = True )
df_all_decks_survived = df_all.groupby(['Deck', 'Survived'] ).count().drop(columns=['Sex', 'Age', 'SibSp', 'Parch', 'Fare',
'Embarked', 'Pclass', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
def get_survived_dist(df):
surv_counts = {'A':{}, 'B':{}, 'C':{}, 'D':{}, 'E':{}, 'F':{}, 'G':{}, 'M':{}}
decks = df.columns.levels[0]
for deck in decks:
for survive in range(0, 2):
surv_counts[deck][survive] = df[deck][survive][0]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
return surv_counts, surv_percentages
def display_surv_dist(percentages):
df_survived_percentages = pd.DataFrame(percentages ).transpose()
deck_names =('A', 'B', 'C', 'D', 'E', 'F', 'G', 'M')
 bar_count = np.arange(len(deck_names))
bar_width = 0.85
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(20, 10))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Deck', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, deck_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage in Decks', size=18, y=1.05)
 plt.show()
all_surv_count, all_surv_per = get_survived_dist(df_all_decks_survived)
 display_surv_dist(all_surv_per )<count_values>
y_transformed = np.log1p(y )
df_all['Deck'] = df_all['Deck'].replace(['A', 'B', 'C'], 'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D', 'E'], 'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F', 'G'], 'FG')
 df_all['Deck'].value_counts()<split>
kf = KFold(n_splits=9, random_state=42, shuffle=True )
df_all.drop(['Cabin'], inplace=True, axis=1)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train, df_test]
for df in dfs:
display_missing(df )<filter>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X,y_true):
rmse = np.sqrt(-cross_val_score(model, X, y_true, scoring="neg_mean_squared_error", cv=kf))
return(rmse )
corr = df_train_corr_nd['Correlation Coefficient'] > 0.1
df_train_corr_nd[corr]<filter>
lasso_reg =make_pipeline(Min() ,Lasso(alpha=0.0005,random_state=44))
corr = df_test_corr_nd['Correlation Coefficient'] > 0.1
df_test_corr_nd[corr]<concatenate>
scores = {}
score = cv_rmse(lasso_reg,x_train,y_transformed)
 print("Lasso: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['Lasso'] =(score.mean() , score.std() )
df_all = concat_df(df_train, df_test)
 df_all.head()<string_transform>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
df_all['Family'] = extract_surname(df_all['Name'])
 df_train = df_all.loc[:890]
df_test = df_all.loc[891:]
dfs = [df_train, df_test]<define_variables>
scores = {}
score = cv_rmse(GBoost,x_train,y_transformed)
 print("gboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gboost'] =(score.mean() , score.std() )
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA<feature_engineering>
model_xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213,
random_state =7, nthread = -1 )
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2<categorify>
scores = {}
score = cv_rmse(model_xgb,x_train,y_transformed)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgboost'] =(score.mean() , score.std() )
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )<categorify>
model_lgb = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )<concatenate>
scores = {}
score = cv_rmse(model_lgb,x_train,y_transformed)
 print("lgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgbm'] =(score.mean() , score.std() )
df_all = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
df_all.drop(columns=drop_cols, inplace=True)
 df_all.head()<prepare_x_and_y>
ridge = make_pipeline(RobustScaler() ,Ridge(alpha = 0.0005,random_state = 12))
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
X_train = pd.DataFrame(X_train)
 y_train = pd.DataFrame(y_train)
 X_test = pd.DataFrame(X_test)
 X_train.columns = feature_stores
X_test.columns = feature_stores
X_train.head()<import_modules>
scores = {}
score = cv_rmse(ridge,x_train,y_transformed)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from xgboost import XGBClassifier
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics<choose_model_class>
bag = BaggingRegressor(base_estimator=model_lgb,n_estimators=2,random_state=2,n_jobs=-1 )
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 old_leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=0)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=20,
max_depth=3,
min_samples_split=2,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=42,
n_jobs=-1,
verbose=0)
 cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0 )<prepare_x_and_y>
scores = {}
score = cv_rmse(bag,x_train,y_transformed)
 print("bag: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['bag'] =(score.mean() , score.std() )
X_rfe = None<train_model>
print('Lightgbm')
 model_lgb.fit(x_train,y_transformed)
 print('model_xgb')
 model_xgb.fit(x_train,y_transformed)
 print('Gradient_boosting')
 GBoost.fit(x_train,y_transformed)
 print('bagging')
 bag.fit(x_train,y_transformed )
<define_variables>
def blend_models_predict(X):
return(
(0.2 * GBoost.predict(X)) + \ 
(0.3 * model_lgb.predict(X)) + \ 
(0.2 * model_xgb.predict(X)) + \ 
(0.3 * bag.predict(np.array(X))))
FEATURES_pipeline = np.array(['Deck_1', 'Deck_4', 'Embarked_2', 'Family_Size_Grouped_1'
'Family_Size_Grouped_3', 'Fare', 'Pclass_1', 'Survival_Rate_NA'
'Ticket_Frequency', 'Title_4'] )<define_variables>
print('RMSLE score on train data:')
 print(rmsle(y_transformed, blend_models_predict(x_train)) )
myrun_bestresults = np.array(['Age', 'Deck_1', 'Deck_2', 'Deck_4', 'Embarked_1', 'Embarked_2',
'Embarked_3', 'Family_Size_Grouped_1', 'Family_Size_Grouped_3',
'Family_Size_Grouped_4', 'Fare', 'Is_Married', 'Pclass_1',
'Pclass_2', 'Pclass_3', 'Sex_1', 'Survival_Rate',
'Survival_Rate_NA', 'Ticket_Frequency', 'Title_2', 'Title_3',
'Title_4'] )<define_variables>
print('Predict submission')
 submission = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] =(np.expm1(blend_models_predict(x_test)) )
choose = [FEATURES,X_rfe,FEATURES_pipeline,myrun_bestresults]
FEATURES = choose[3]<choose_model_class>
submission.to_csv("submission2.csv", index=False )
<init_hyperparams><EOS> 
q1 = submission['SalePrice'].quantile(0.0042)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission3.csv", index=False )