Kaggle competition: House Prices - Advanced Regression Techniques
warnings.filterwarnings('ignore' )<load_from_csv>
import catboost
import optuna
import imblearn
from catboost import CatBoostRegressor
from imblearn.under_sampling import RandomUnderSampler
import numpy as np
import pandas as pd
from catboost import *
import matplotlib.pyplot as plt
import seaborn as sns
from catboost import Pool
from datetime import datetime
from numpy import mean
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split,cross_val_score
from sklearn.linear_model import LinearRegression,RidgeCV
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report, accuracy_score, roc_auc_score
from scipy.stats import norm,skew
from scipy import stats
from sklearn.metrics import mean_squared_error,make_scorer
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from tqdm import tqdm
import pandas as pd
import nltk
import operator
import re
import sys
from scipy import stats
from nltk.corpus import stopwords
import nltk
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
import statsmodels.api as sm
from statsmodels.formula.api import ols
import time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )<create_dataframe>
Train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 Test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
params = [{'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]}]
reg_params = np.zeros(512)
 for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=25, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
qda = QuadraticDiscriminantAnalysis()
clf = GridSearchCV(qda, params, cv=4)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 reg_params[i] = clf.best_params_['reg_param']
oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
<train_on_grid>
pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )
train_f = train.copy()
train_f.loc[oof > 0.995, 'target'] = 1
train_f.loc[oof < 0.005, 'target'] = 0
oof_ls = np.zeros(len(train))
pred_te_ls = np.zeros(len(test))
for k in tqdm_notebook(range(512)) :
train2 = train_f[train_f['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2['target']=-1
train2p = pd.concat([train2,test2],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train4p = sel.transform(train2p[cols])
 train4 = sel.transform(train2[cols])
 test4 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=25, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train4p, train2p['target']):
test_index3 = test_index[ test_index<len(train4)]
clf = LabelSpreading(gamma=0.01,kernel='rbf', max_iter=10,n_jobs=-1)
 clf.fit(train4p[train_index,:],train2p.loc[train_index]['target'])
 oof_ls[idx1[test_index3]] = clf.predict_proba(train4[test_index3,:])[:,1]
pred_te_ls[test2.index] += clf.predict_proba(test4)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_ls)
 print('CV for LabelSpreading =',round(auc,5))<feature_engineering>
Train['MSSubClass']=Train['MSSubClass'].astype('int')
 Train['YrSold']=2020-Train['YrSold']
Train['MoSold']=Train['MoSold'].astype('str')
 Train['GarageYrBlt']=2020-Train['GarageYrBlt']
Train['YearBuilt']=2020-Train['YearBuilt']
Train['YearRemodAdd']=2020-Train['YearRemodAdd']
for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
usefull_pred_te_ls = pred_te_ls[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 new_oof_ls = np.hstack([oof_ls, usefull_pred_te_ls])
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx0 = new_train[new_train['wheezy-copper-turtle-magic']==i].index
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
train4 = np.hstack([train3,np.array([new_oof_ls[idx0]] ).T])
 test4 = np.hstack([test3,np.array([pred_te_ls[idx2]] ).T])
 skf = StratifiedKFold(n_splits=25, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train4[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train4[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test4)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}')
<save_to_csv>
Test['MSSubClass']=Test['MSSubClass'].astype('int')
 Test['YrSold']=2020-Test['YrSold']
Test['MoSold']=Test['MoSold'].astype('str')
 Test['GarageYrBlt']=2020-Test['GarageYrBlt']
Test['YearBuilt']=2020-Test['YearBuilt']
Test['YearRemodAdd']=2020-Test['YearRemodAdd']
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<set_options>
[print(i,Train[i].unique() [:10],len(Train[i].unique()),Train[i].dtype)for i in Train.columns]
warnings.filterwarnings('ignore' )<load_from_csv>
numeric_feats = Train.dtypes[Train.dtypes ==("float" or "int")].index
skewed_feats = Train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
print(skewed_feats)
 Train[skewed_feats] = np.log1p(Train[skewed_feats])
 Test[skewed_feats] = np.log1p(Test[skewed_feats] )
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )<define_search_space>
Train=Train.fillna(0)
 Test=Test.fillna(0 )
reg_params = [0.01,0.23,0.01,0.08,0.27,0.03,0.16,0.33,0.28,0.02,0.01,0.07,0.01,0.31
,0.28,0.26,0.02,0.1,0.11,0.17,0.42,0.34,0.11,0.22,0.61,0.12,0.14,0.19
,0.02,0.01,0.02,0.06,0.36,0.09,0.11,0.01,0.03,0.11,0.01,0.11,0.23,0.02
,0.17,0.14,0.12,0.06,0.08,0.01,0.26,0.01,0.02,0.04,0.33,0.03,0.07,0.16
,0.01,0.02,0.28,0.17,0.04,0.19,0.05,0.11,0.03,0.22,0.17,0.16,0.06,0.14
,0.01,0.07,0.05,0.06,0.2,0.05,0.02,0.09,0.05,0.01,0.08,0.1,0.01,0.22
,0.19,0.13,0.02,0.03,0.12,0.07,0.15,0.09,0.02,0.16,0.01,0.39,0.12,0.14
,0.09,0.41,0.32,0.13,0.15,0.01,0.41,0.12,0.01,0.13,0.17,0.03,0.08,0.12
,0.21,0.01,0.16,0.28,0.01,0.02,0.11,0.21,0.27,0.03,0.17,0.05,0.02,0.4
,0.28,0.01,0.03,0.28,0.02,0.12,0.17,0.1,0.01,0.1,0.01,0.24,0.11,0.18
,0.17,0.08,0.1,0.04,0.13,0.04,0.12,0.25,0.13,0.19,0.05,0.03,0.1,0.01
,0.05,0.01,0.01,0.01,0.18,0.24,0.13,0.01,0.15,0.01,0.09,0.04,0.06,0.01
,0.06,0.11,0.21,0.08,0.21,0.04,0.04,0.04,0.2,0.01,0.19,0.14,0.11,0.03
,0.23,0.01,0.04,0.13,0.46,0.04,0.07,0.08,0.07,0.01,0.11,0.2,0.07,0.23
,0.2,0.14,0.07,0.06,0.16,0.02,0.33,0.13,0.11,0.06,0.22,0.22,0.19,0.03
,0.1,0.37,0.22,0.01,0.01,0.22,0.07,0.01,0.23,0.35,0.03,0.29,0.01,0.04
,0.01,0.04,0.07,0.23,0.2,0.09,0.01,0.23,0.3,0.06,0.09,0.04,0.46,0.25
,0.14,0.01,0.04,0.03,0.01,0.04,0.11,0.08,0.01,0.09,0.05,0.1,0.05,0.28
,0.02,0.08,0.01,0.06,0.38,0.04,0.01,0.15,0.21,0.01,0.01,0.45,0.18,0.27
,0.24,0.01,0.04,0.14,0.13,0.18,0.22,0.32,0.13,0.07,0.26,0.17,0.12,0.14
,0.09,0.13,0.08,0.09,0.07,0.01,0.02,0.04,0.01,0.07,0.32,0.01,0.36,0.09
,0.11,0.06,0.46,0.11,0.16,0.21,0.01,0.1,0.01,0.1,0.23,0.05,0.33,0.01
,0.24,0.04,0.01,0.04,0.1,0.01,0.36,0.44,0.03,0.08,0.21,0.01,0.18,0.01
,0.17,0.19,0.03,0.01,0.18,0.15,0.48,0.06,0.17,0.18,0.37,0.01,0.31,0.01
,0.16,0.18,0.11,0.08,0.08,0.07,0.28,0.02,0.09,0.08,0.01,0.09,0.01,0.07
,0.01,0.24,0.09,0.02,0.37,0.16,0.04,0.14,0.22,0.06,0.29,0.16,0.06,0.06
,0.04,0.05,0.25,0.07,0.01,0.01,0.21,0.02,0.04,0.3,0.39,0.02,0.23,0.22
,0.05,0.01,0.06,0.05,0.02,0.01,0.02,0.12,0.11,0.05,0.01,0.2,0.01,0.08
,0.08,0.04,0.33,0.06,0.16,0.35,0.18,0.13,0.01,0.01,0.12,0.18,0.01,0.01
,0.17,0.08,0.48,0.18,0.01,0.02,0.35,0.15,0.34,0.01,0.14,0.01,0.32,0.34
,0.15,0.1,0.18,0.18,0.11,0.24,0.01,0.13,0.03,0.36,0.01,0.08,0.01,0.13
,0.07,0.08,0.24,0.01,0.05,0.05,0.1,0.07,0.21,0.01,0.08,0.11,0.09,0.04
,0.01,0.11,0.02,0.09,0.16,0.51,0.17,0.09,0.03,0.12,0.06,0.18,0.01,0.01
,0.02,0.02,0.01,0.27,0.28,0.09,0.02,0.13,0.03,0.16,0.15,0.04,0.17,0.19
,0.26,0.01,0.04,0.1,0.04,0.01,0.07,0.05,0.02,0.59,0.01,0.2,0.12,0.1
,0.07,0.18,0.08,0.01,0.34,0.15,0.28,0.12,0.05,0.01,0.11,0.08,0.12,0.04
,0.05,0.33,0.01,0.01,0.09,0.07,0.19,0.09]<create_dataframe>
X_t=Train[Train.columns[Train.columns!='SalePrice']]
y_t=Train['SalePrice']
X_test=Test
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=30, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
<feature_engineering>
X_train,X_valid,y_train,y_valid=train_test_split(X_t,y_t,test_size=0.2,shuffle=True )
for i in range(5):
test['target'] = preds
test.loc[test['target'] > 0.94, 'target'] = 1
test.loc[test['target'] < 0.06, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0])
 new_train.loc[oof > 0.98, 'target'] = 1
new_train.loc[oof < 0.02, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=30, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}' )<save_to_csv>
categorical_features_indices= np.where(X_test.dtypes == np.object)[0]
Train_set=Pool(X_train, y_train,cat_features=categorical_features_indices)
 Eval_set=Pool(X_valid, y_valid,cat_features=categorical_features_indices)
 def objective(trial):
param = {
'iterations':500,
'learning_rate':0.05,
'use_best_model':True,
'od_type' : "Iter",
'od_wait' : 100,
"depth": trial.suggest_int("max_depth", 2, 10),
"l2_leaf_reg": trial.suggest_loguniform("lambda", 1e-8, 100),
'eval_metric':trial.suggest_categorical("loss_function",['MAE','RMSE']),
'one_hot_max_size':1024
}
model=CatBoostRegressor(**param)
 print(param)
 model.fit(Train_set,eval_set=Eval_set,plot=False,verbose=False)
 pred=model.predict(Pool(X_valid,cat_features= np.where(X_valid.dtypes == np.object)[0]))
mae=sklearn.metrics.mean_absolute_error(pred,y_valid)
 return mae
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<import_modules>
import sklearn
warnings.filterwarnings('ignore' )<load_from_csv>
param={'one_hot_max_size':1024,'iterations': 2000, 'learning_rate': 0.05, 'use_best_model': True, 'od_type': 'Iter', 'od_wait': 100, 'depth': 7, 'l2_leaf_reg': 0.13679400187948992,'loss_function':'MAE'}
model1=CatBoostRegressor(**param)
 model1.fit(Train_set,eval_set=Eval_set,plot=True,verbose=False)
 pred=model1.predict(Pool(X_test,cat_features= np.where(X_test.dtypes == np.object)[0]))
def load_data(data):
return pd.read_csv(data)
 with multiprocessing.Pool() as pool:
train, test, sub = pool.map(load_data, ['.. /input/train.csv', '.. /input/test.csv', '.. /input/sample_submission.csv'] )<define_variables>
X_test['SalePrice']=pred
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<define_variables>
X_test[['Id','SalePrice']].to_csv('submission.csv',index=False )