Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
import numpy as np
import pandas as pd
import os
import seaborn as sns
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.ensemble import VotingClassifier
import torch
from torch import nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import math
from torch.utils.data import TensorDataset, DataLoader
import matplotlib.pyplot as plt
from tqdm import tqdm
import catboost
from sklearn.metrics import mean_squared_error
import scipy
from scipy import stats
from scipy.stats import norm
gridBoost.fit(X_train,y_train)
 print(gridBoost.best_score_)
 print(gridBoost.best_estimator_ )<train_model>
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', sep=',' )
gridBoost.best_estimator_.fit(X_train,y_train)
 predict=gridBoost.best_estimator_.predict(X_test)
 print(accuracy_score(y_test,predict)) 
<train_model>
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
xgb=XGBClassifier(max_depth=2, n_estimators=700, learning_rate=0.009,nthread=-1,subsample=1,colsample_bytree=0.8)
 xgb.fit(X_train,y_train)
 predict=xgb.predict(X_test)
 print(accuracy_score(y_test,predict))
print(confusion_matrix(y_test,predict))
print(precision_score(y_test,predict))
print(recall_score(y_test,predict))<compute_train_metric>
target = 'SalePrice'
print(train.loc[:, target].isnull().any())
(mu, sigma)= norm.fit(train['SalePrice'])
 print(f'mu = {mu:.2f} and sigma = {sigma:.2f}' )
lda=LinearDiscriminantAnalysis()
lda.fit(X_train,y_train)
 predict=lda.predict(X_test)
 print(accuracy_score(y_test,predict))
print(precision_score(y_test,predict))
print(recall_score(y_test,predict))<train_model>
sns.set_style('whitegrid' )
dectree = DecisionTreeClassifier(criterion="entropy",
max_depth=5,
class_weight = 'balanced',
min_weight_fraction_leaf = 0.009,
random_state=2000)
 dectree.fit(X_train, y_train)
 y_pred = dectree.predict(X_test)
 dectree_accy = round(accuracy_score(y_pred, y_test), 3)
 print(dectree_accy)
 print(confusion_matrix(y_test,y_pred))
print(precision_score(y_test,y_pred))
print(recall_score(y_test,y_pred)) 
<choose_model_class>
train['Alley'] = train['Alley'].fillna('No alley access')
 train['BsmtQual'] = train['BsmtQual'].fillna('No Basement')
 train['BsmtCond'] = train['BsmtCond'].fillna('No Basement')
 train['BsmtExposure'] = train['BsmtExposure'].fillna('No Basement')
 train['BsmtFinType1'] = train['BsmtFinType1'].fillna('No Basement')
 train['BsmtFinType2'] = train['BsmtFinType2'].fillna('No Basement')
 train['FireplaceQu'] = train['FireplaceQu'].fillna('No Fireplace')
 train['GarageType'] = train['GarageType'].fillna('No Garage')
 train['GarageFinish'] = train['GarageFinish'].fillna('No Garage')
 train['GarageQual'] = train['GarageQual'].fillna('No Garage')
 train['GarageCond'] = train['GarageCond'].fillna('No Garage')
 train['PoolQC'] = train['PoolQC'].fillna('No Pool')
 train['Fence'] = train['Fence'].fillna('No Fence')
 train['MiscFeature'] = train['MiscFeature'].fillna('None')
 test['Alley'] = test['Alley'].fillna('No alley access')
 test['BsmtQual'] = test['BsmtQual'].fillna('No Basement')
 test['BsmtCond'] = test['BsmtCond'].fillna('No Basement')
 test['BsmtExposure'] = test['BsmtExposure'].fillna('No Basement')
 test['BsmtFinType1'] = test['BsmtFinType1'].fillna('No Basement')
 test['BsmtFinType2'] = test['BsmtFinType2'].fillna('No Basement')
 test['FireplaceQu'] = test['FireplaceQu'].fillna('No Fireplace')
 test['GarageType'] = test['GarageType'].fillna('No Garage')
 test['GarageFinish'] = test['GarageFinish'].fillna('No Garage')
 test['GarageQual'] = test['GarageQual'].fillna('No Garage')
 test['GarageCond'] = test['GarageCond'].fillna('No Garage')
 test['PoolQC'] = test['PoolQC'].fillna('No Pool')
 test['Fence'] = test['Fence'].fillna('No Fence')
 test['MiscFeature'] = test['MiscFeature'].fillna('None' )
randomforest = RandomForestClassifier(n_estimators=100,max_depth=5,min_samples_split=20,max_features=0.2, min_samples_leaf=8,random_state=20)
 randomforest.fit(X_train, y_train)
 y_pred = randomforest.predict(X_test)
 random_accy = round(accuracy_score(y_pred, y_test), 3)
 print(random_accy)
 print(precision_score(y_test,y_pred))
print(recall_score(y_test,y_pred))
print(confusion_matrix(y_test,y_pred)) 
<train_model>
train['Pool_presence'] = [0 if val=='No Pool' else 1 for val in train['PoolQC']]
test['Pool_presence'] = [0 if val=='No Pool' else 1 for val in test['PoolQC']]
train['CentralAir'] = [1 if x == train['CentralAir'].unique() [0] else 0 for x in train.loc[:,'CentralAir'].values]
test['CentralAir'] = [1 if x == test['CentralAir'].unique() [0] else 0 for x in test.loc[:,'CentralAir'].values]
BaggingClassifier = BaggingClassifier()
BaggingClassifier.fit(X_train, y_train)
 y_pred = BaggingClassifier.predict(X_test)
 bagging_accy = round(accuracy_score(y_pred, y_test), 3)
 print(bagging_accy )<predict_on_test>
from catboost.eval.catboost_evaluation import *
from catboost.utils import create_cd
voting_classifier = VotingClassifier(estimators=[ 
('logreg',logreg), 
('random_forest', randomforest), 
('decision_tree',dectree), 
('XGB Classifier', xgb), 
('BaggingClassifier', BaggingClassifier)])
 voting_classifier.fit(X_train,y_train)
 y_pred = voting_classifier.predict(X_test)
 voting_accy = round(accuracy_score(y_pred, y_test), 3)
 print(voting_accy )<predict_on_test>
cols = list(train.columns)
 a, b = cols.index('SalePrice'), cols.index('Pool_presence')
 cols[b], cols[a] = cols[a], cols[b]
train = train[cols]
train.head()
model = CatBoostClassifier(verbose=False, one_hot_max_size=3)
 model.fit(X_train,y_train)
 y_pred=model.predict(X_test)
 acc = round(accuracy_score(y_pred, y_test), 3)
 print(acc )<predict_on_test>
train.to_csv('train2.csv', index=False )
y_predict=model.predict(testframe )<save_to_csv>
feature_names = dict(list(enumerate(train.keys())))
 cat_dict = {i:label for i,label in enumerate(train.columns)if label in cat_features+['Fence', 'MiscFeature', 'Alley', 'PoolQC']}
del feature_names[80]
temp = pd.DataFrame(pd.DataFrame({
"PassengerId": passenger_id,
"Survived": y_predict
}))
temp.to_csv(".. /working/submission3.csv", index = False )<set_options>
fold_size = int(round(len(train)/2, -2))
folds_count=10
description_file='./train.cd'
train2_file='./train2.csv'
learn_params={'iterations':505,
'task_type' : 'GPU',
'random_seed': 2,
'learning_rate' : 0.1,
'max_depth': 10,
'l2_leaf_reg': 9.8,
'loss_function': 'RMSE',
'max_ctr_complexity' : 2,
'logging_level': 'Silent',
'boosting_type': 'Plain',}
evaluator = CatboostEvaluation(train2_file,
fold_size,
folds_count,
delimiter=',',
column_description=description_file,
partition_random_seed=2,
has_header=True, 
)
 result = evaluator.eval_features(learn_config=learn_params,
eval_metrics=["RMSE"],
features_to_eval=range(1,80),
)
warnings.filterwarnings("ignore")
<load_from_csv>
logloss_result = result.get_metric_results("RMSE")
 logloss_result.get_baseline_comparison()
result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 result = result.drop(columns=['WLoc', 'NumOT', 'DayNum'])
 result.head()<load_from_csv>
good_features = logloss_result.get_baseline_comparison() [logloss_result.get_baseline_comparison() ['Decision']=='GOOD'].index
good_features = [pd.read_csv(train2_file ).columns[int(str(feature)[-2:])] for i,feature in enumerate(good_features)]
len(good_features), good_features
seeds = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')
 seeds.Seed = seeds.Seed.map(lambda string : int(string[1:3]))
seeds.head()<load_from_csv>
train = train[good_features+[target]]
test = test[good_features]
team_name = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MTeamSpellings.csv',
encoding='cp1252')
 team_name = team_name.drop_duplicates(subset=['TeamID'], keep='last' ).reset_index(drop=True)
 team_name.TeamNameSpelling = team_name.TeamNameSpelling.map(lambda string : string.upper())
 team_name.head()<merge>
def empty_features(test, train, threshhold=0, verbose=False):
if verbose:
print(' column \t test \t\t train
', '*'*40)
 i=1
empty_list=[]
for col in test.columns:
percentage_train =(train[col].isnull().sum() /len(train)) *100
percentage_test =(test[col].isnull().sum() /len(test)) *100
if percentage_train and percentage_test:
if(( percentage_test>=threshhold)|(percentage_train>=threshhold)) :
empty_list.append(col)
 if verbose:
print(i,'{}{}% \t{}%'.format(col.ljust(15,' '), round(percentage_test, 3), round(percentage_train,3)))
 i+=1
return empty_list
Wname = team_name.rename(columns={'TeamNameSpelling':'Wteam_name', 'TeamID':'WTeamID'})
 team_result = pd.merge(left=result, right=Wname, how='left', on=['WTeamID'])
 win_by_year = team_result.groupby(['Season', 'Wteam_name'])\
['WScore'].count().reset_index() \ 
.rename(columns={'Wteam_name':'Team', 'WScore':'Win_matches'})
 df = win_by_year[win_by_year.Win_matches == win_by_year.Win_matches.max() ].drop(columns=['Win_matches'] ).set_index('Season')
 df<feature_engineering>
empty_features_list=empty_features(test=test, train=train, verbose=True, threshhold=80)
 empty_features_list
win_by_year = win_by_year.groupby(['Team', 'Season'] ).Win_matches.sum() \ 
.unstack(fill_value=0 ).cumsum(axis=1 ).sort_values(by=2019, ascending=False ).head(15)
 cum_win = pd.melt(win_by_year, value_vars=win_by_year.columns, value_name="Win_matches")
 cum_win['Team'] = list(win_by_year.index)*35
cum_win = cum_win[['Season', 'Team', 'Win_matches']]<merge>
only_1_value = [feature for feature in train.nunique().index if train.nunique() [feature]==1]
only_1_value
Wseeds = seeds.rename(columns={'TeamID':'WTeamID', 'Seed':'WSeed'})
 Lseeds = seeds.rename(columns={'TeamID':'LTeamID', 'Seed':'LSeed'})
 data = pd.merge(left=result, right=Wseeds, how='left', on=['Season', 'WTeamID'])
 data = pd.merge(left=data, right=Lseeds, on=['Season', 'LTeamID'])
 data.head()<load_from_csv>
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
drop_by_corr = [column for column in upper.columns if any(upper[column] > 0.875)]
print(drop_by_corr )
scores = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')
 scores.head()<groupby>
def quasi_constant(df, threshold=0, verbose=True):
features_list = []
for feature in df:
table = df[feature].value_counts() / np.float(len(df))
first=1
for name in table.index:
if table[name]>=threshold:
if first:
if verbose:
print(f'for feature "{feature}":', 'value\t\tscore', sep='
')
 first=0
if verbose:
print(name, table[name], sep='\t', end='
'+'*'*50+'
')
 if not first:
features_list.append(feature)
 return features_list
season_score = result_scores.groupby(['Season', 'TeamID'])['Score'].sum()
season_score.sort_values(ascending=False ).head()<sort_values>
quasi_constant_features = quasi_constant(train, 0.95)
 quasi_constant_features
result_scores.sort_values(by='Score', ascending=False ).head(15 )<filter>
bad_features = drop_by_corr+only_1_value+quasi_constant_features+empty_features_list
len(bad_features), bad_features
team_name[team_name.TeamID.isin([1258, 1328])]<merge>
good_features = [feature for feature in good_features if feature not in bad_features]
len(good_features), good_features
data = pd.merge(data, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 data = data.rename(columns={'Score':'WScoreT'})
 data = pd.merge(data, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 data = data.rename(columns={'Score':'LScoreT'})
 data = data.drop(columns=['WScore', 'LScore'])
 data.head()<drop_column>
train = train[good_features+[target]]
test = test[good_features]
Wdata = data.drop(columns=['Season',
])
 Wdata.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2',
'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2',
'WTeamID':'TeamID_1', 'LTeamID': 'TeamID_2'}, inplace=True)
 Wdata.head()<rename_columns>
len(train.columns), len(pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', sep=',' ).columns )
Ldata = data[['LTeamID', 'WTeamID', 'LSeed', 'WSeed', 'LScoreT', 'WScoreT']]
Ldata.rename(columns={'LTeamID':'TeamID_1', 'WTeamID':'TeamID_2',
'LSeed':'Seed1', 'WSeed':'Seed2',
'LScoreT':'ScoreT1', 'WScoreT':'ScoreT2',}, inplace=True)
 Ldata.head()<feature_engineering>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2 )
Wdata['Seed_diff'] = Wdata['Seed1'] - Wdata['Seed2']
Wdata['ScoreT_diff'] = Wdata['ScoreT1'] - Wdata['ScoreT2']
Ldata['Seed_diff'] = Ldata['Seed1'] - Ldata['Seed2']
Ldata['ScoreT_diff'] = Ldata['ScoreT1'] - Ldata['ScoreT2']<feature_engineering>
X,y = pipelining_preprocessor(train, filler = 'out_of_range', target='SalePrice', type='cat')
 y = np.log1p(y )
Wdata['result'] = 1
Ldata['result'] = 0
train = pd.concat(( Wdata, Ldata)).reset_index(drop=True)
 train.head()<feature_engineering>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, shuffle=True, random_state=2 )
test = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 test['Season'] = test.ID.map(lambda string : int(string.split('_')[0]))
test['TeamID_1'] = test.ID.map(lambda string : int(string.split('_')[1]))
test['TeamID_2'] = test.ID.map(lambda string : int(string.split('_')[2]))
test = test.drop(columns=['ID'])
 test.head()<merge>
cats = list(train.select_dtypes(include='object' ).columns )
test = pd.merge(test, seeds, left_on=['Season', 'TeamID_1'], right_on=['Season', 'TeamID'], how='left')
 test.rename(columns={'Seed':'Seed1'}, inplace=True)
 test = test.drop('TeamID', axis=1)
 test = pd.merge(test, seeds, left_on=['Season', 'TeamID_2'], right_on=['Season', 'TeamID'], how='left')
 test.rename(columns={'Seed':'Seed2'}, inplace=True)
 test = test.drop('TeamID', axis=1)
 test = pd.merge(test, season_score, left_on=['Season', 'TeamID_1'], right_on=['Season', 'TeamID'], how='left')
 test.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test = pd.merge(test, season_score, left_on=['Season', 'TeamID_2'], right_on=['Season', 'TeamID'], how='left')
 test.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test['Seed_diff'] = test['Seed1'] - test['Seed2']
test['ScoreT_diff'] = test['ScoreT1'] - test['ScoreT2']
test = test.drop(columns=['Pred', 'Season'])
 test.head()<prepare_x_and_y>
full_pool = catboost.Pool(X, label = y, cat_features = cats)
 train_pool = catboost.Pool(X_train, label = y_train, cat_features = cats)
 val_pool = catboost.Pool(X_val, label = y_val, cat_features = cats)
 test_pool = catboost.Pool(pipelining_preprocessor(test[X.columns], filler = 'out_of_range', type='cat'), cat_features = cats )
X_train = train.drop(columns=['result'])
 y_train = train.result
X_test = test.copy()
data_full = pd.concat([X_train, X_test])
 data_full.shape<categorify>
cat_params = {
}
cat_model = catboost.CatBoostRegressor(loss_function='RMSE',
random_seed=2,
max_depth = 10,
learning_rate = 0.05,
random_strength=1,
max_ctr_complexity=1,
l2_leaf_reg=8,
grow_policy = 'Lossguide',
task_type='GPU', 
)
 grid_search_results = cat_model.grid_search(cat_params, full_pool,
partition_random_seed=2, cv = skf,
search_by_train_test_split=True,
plot=True)
 grid_search_results['params']
OH_cols = ['TeamID_1', 'TeamID_2']
OH_full = pd.get_dummies(
 data_full[OH_cols],
columns=OH_cols,
drop_first=True,
dummy_na=True,
sparse=True,
).sparse.to_coo()<drop_column>
cat_model = catboost.CatBoostRegressor(loss_function='RMSE',
random_seed=2,
max_depth = 10,
learning_rate = 0.05,
random_strength=1,
max_ctr_complexity=1,
l2_leaf_reg=8,
grow_policy = 'Lossguide',
task_type='GPU',
)
retain_full = data_full.drop(columns=OH_cols)
 retain_full = retain_full/retain_full.max()
retain_full.head()<categorify>
cat_model.fit(train_pool, eval_set=val_pool )
encoded_full = scipy.sparse.hstack([OH_full, retain_full, retain_full**2] ).tocsr()
print(encoded_full.shape)
 encoded_train = encoded_full[:len(X_train)]
encoded_test = encoded_full[len(X_train):]<load_from_csv>
import shap
submission = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 submission.head()<choose_model_class>
vals= np.abs(shap_values ).mean(0)
 feature_importance_shap = pd.DataFrame(list(zip(X_train.columns, vals)) , columns=['feature','feature_importance_shap'])
 feature_importance_shap.sort_values(by=['feature_importance_shap'], ascending=False, inplace=True)
 feature_importance_shap.reset_index(inplace=True, drop=True)
 feature_importance_shap
<compute_train_metric><EOS> 
gb_pred = list(map(int, map(np.expm1, cat_model.predict(test_pool))))
gb_output = pd.DataFrame({'Id': pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')['Id'],
'SalePrice': gb_pred})
 gb_output.to_csv('submission_catboost.csv', index=False)
 gb_output