Kaggle competition: House Prices - Advanced Regression Techniques
def oof_distribution(tr, cols):
ncols = 3
nrows = int(np.ceil(len(cols)/ 3))
figsize =(12, 4 * nrows)
 fig, axes = plt.subplots(ncols=ncols, nrows=nrows, figsize=figsize)
 for col, ax in zip(cols, axes.ravel()):
auc = roc_auc_score(tr['target'], tr[col])
 tr_0 = tr[tr['target'] == 0]
tr_1 = tr[tr['target'] == 1]
tr_0 = tr_0[tr_0[col] > 0.0]
tr_0 = tr_0[tr_0[col] < 1.0]
tr_1 = tr_1[tr_1[col] > 0.0]
tr_1 = tr_1[tr_1[col] < 1.0]
tr_0 = tr_0.sample(5000)
 tr_1 = tr_1.sample(5000)
 sns.kdeplot(tr_0[col].values, ax=ax, shade=True, color="b", label='0')
 sns.kdeplot(tr_1[col].values, ax=ax, shade=True, color="r", label='1')
 ax.set_title('{} AUC: {:.5f}'.format(col, auc))
plt.tight_layout()
plt.show()<save_to_csv>
%config InlineBackend.figure_format = 'retina'
%matplotlib inline
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit['target'] = te['Stack_LR_Mean'].values
submit.to_csv('submission.csv', index=False )<load_from_csv>
train_set = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', header=0)
 test_set =pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', header=0 )
warnings.filterwarnings('ignore')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = 0
sub.to_csv('submission.csv',index=False)
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 if len(test)< 150000:
[].shape
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )<create_dataframe>
full_data = pd.concat(( train_set.loc[:,'MSSubClass':'YrSold'],test_set.loc[:,'MSSubClass':'YrSold']))
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added")
 new_train.loc[oof > 0.99, 'target'] = 1
new_train.loc[oof < 0.01, 'target'] = 0<find_best_model_class>
train_set["SalePrice"] = np.log1p(train_set["SalePrice"])
 numeric_feats = full_data.dtypes[full_data.dtypes != "object"].index
skewed_feats = train_set[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
full_data[skewed_feats] = np.log1p(full_data[skewed_feats] )
def count(x):
ret = [0,0,0,0,0,0]
for i in x:
ret[i] += 1
return ret
def get_clf(data, data_pos, data_neg):
start = time.time()
while True:
clf = bgm(n_components=6, covariance_type="full")
 clf.fit(data)
 p1 = count(clf.predict(data_pos))
p2 = count(clf.predict(data_neg))
flag = True
up = len(data_pos)/3.6
low = len(data_pos)/30
for i in range(6):
flag &=(p1[i] > up and p2[i] < low)or(p2[i] > up and p1[i] < low)
 t = time.time() - start
if t > 40:
return None, None
if flag:
pindex = set()
for i in range(6):
if p1[i] > 99:
pindex.add(i)
 return pindex, clf<train_on_grid>
full_data = pd.get_dummies(full_data )
poof = np.zeros(len(train))
ppreds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
ptrain = train[train['wheezy-copper-turtle-magic']==i]
ptrain_add = new_train[new_train['wheezy-copper-turtle-magic']==i]
ptest = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test[test['wheezy-copper-turtle-magic']==i].index
ptrain_add.reset_index(drop=True,inplace=True)
 ptest.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(ptrain[cols])
 ptrain_np = sel.transform(ptrain[cols])
 ptrain_add_np = sel.transform(ptrain_add[cols])
 ptest_np = sel.transform(ptest[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(ptrain_add_np, ptrain_add['target']):
test_index3 = test_index[ test_index<len(ptrain)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(ptrain_add_np[train_index,:],ptrain_add.loc[train_index]['target'])
 poof[idx1[test_index3]] = clf.predict_proba(ptrain_add_np[test_index3,:])[:,1]
ppreds[idx2] += clf.predict_proba(ptest_np)[:,1] / skf.n_splits
auc1 = roc_auc_score(ptrain['target'], poof[idx1])
 print(round(auc1,5))
ptrain_add_pos = ptrain_add[ptrain_add['target'] == 1]
ptrain_add_neg = ptrain_add[ptrain_add['target'] == 0]
ptrain_add_pos_np = sel.transform(ptrain_add_pos[cols])
 ptrain_add_neg_np = sel.transform(ptrain_add_neg[cols])
 pindex, clf = get_clf(ptrain_add_np, ptrain_add_pos_np, ptrain_add_neg_np)
 if clf:
ppp = clf.predict(ptrain_np)
 pppt = clf.predict(ptest_np)
 pr = clf.predict_proba(ptest_np)
 preds = []
pt = []
for k in ppp:
if k in pindex:
preds.append(1)
 else:
preds.append(0)
 for i, k in enumerate(pppt):
if k in pindex:
pt.append(pr[i][k])
 else:
pt.append(1 - pr[i][k])
 auc2 = roc_auc_score(ptrain['target'],preds)
 poof[idx1] = preds
ppreds[idx2] = pt
print(round(auc1,5),round(auc2,5))
auc = roc_auc_score(train['target'],poof)
 print('QDA scores CV =',round(auc,5))<feature_engineering>
full_data = full_data.fillna(full_data.mean() )
test['target'] = ppreds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added")
 new_train.loc[poof > 0.99, 'target'] = 1
new_train.loc[poof < 0.01, 'target'] = 0
poof = np.zeros(len(train))
ppreds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
ptrain = train[train['wheezy-copper-turtle-magic']==i]
ptrain_add = new_train[new_train['wheezy-copper-turtle-magic']==i]
ptest = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test[test['wheezy-copper-turtle-magic']==i].index
ptrain_add.reset_index(drop=True,inplace=True)
 ptest.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(ptrain[cols])
 ptrain_np = sel.transform(ptrain[cols])
 ptrain_add_np = sel.transform(ptrain_add[cols])
 ptest_np = sel.transform(ptest[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(ptrain_add_np, ptrain_add['target']):
test_index3 = test_index[ test_index<len(ptrain)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(ptrain_add_np[train_index,:],ptrain_add.loc[train_index]['target'])
 poof[idx1[test_index3]] = clf.predict_proba(ptrain_add_np[test_index3,:])[:,1]
ppreds[idx2] += clf.predict_proba(ptest_np)[:,1] / skf.n_splits
auc1 = roc_auc_score(ptrain['target'], poof[idx1])
 print(round(auc1,5))
ptrain_add_pos = ptrain_add[ptrain_add['target'] == 1]
ptrain_add_neg = ptrain_add[ptrain_add['target'] == 0]
ptrain_add_pos_np = sel.transform(ptrain_add_pos[cols])
 ptrain_add_neg_np = sel.transform(ptrain_add_neg[cols])
 pindex, clf = get_clf(ptrain_add_np, ptrain_add_pos_np, ptrain_add_neg_np)
 if clf:
ppp = clf.predict(ptrain_np)
 pppt = clf.predict(ptest_np)
 pr = clf.predict_proba(ptest_np)
 preds = []
pt = []
for k in ppp:
if k in pindex:
preds.append(1)
 else:
preds.append(0)
 for i, k in enumerate(pppt):
if k in pindex:
pt.append(pr[i][k])
 else:
pt.append(1 - pr[i][k])
 auc2 = roc_auc_score(ptrain['target'],preds)
 if auc1 - auc2 > 0.004:
poof[idx1] = preds
ppreds[idx2] =(np.array(pt)+ ppreds[idx2])/ 2
else:
ppreds[idx2] = pt
print(round(auc1,5),round(auc2,5))
auc = roc_auc_score(train['target'],poof)
 print('QDA scores CV =',round(auc,5))<save_to_csv>
x_train = full_data[:train_set.shape[0]]
x_test = full_data[train_set.shape[0]:]
y = train_set.SalePrice
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = ppreds
sub.to_csv('submission.csv',index=False)
 plt.hist(ppreds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<set_options>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, x_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )
%matplotlib inline
warnings.filterwarnings('ignore')
<load_from_csv>
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(x_train, y )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )<categorify>
r = rmse_cv(model_lasso ).mean()
print(r )
def get_mean_cov(x,y):
ones =(y==1 ).astype(bool)
 x2 = x[ones]
gmm_1 = GaussianMixture(n_components=2)
 gmm_1.fit(x2)
 m1 = gmm_1.means_
p1 = gmm_1.precisions_
zeros =(y==0 ).astype(bool)
 x2b = x[zeros]
gmm_0 = GaussianMixture(n_components=2)
 gmm_0.fit(x2b)
 m2 = gmm_0.means_
p2 = gmm_0.precisions_
ms = np.concatenate(( m1,m2),axis=0)
 ps = np.concatenate(( p1,p2),axis=0)
 return ms,ps<train_model>
model_ridge = Ridge()
np.random.seed(42)
 oof_gmm = np.zeros(len(train))
preds_gmm = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) , 
('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=2019,shuffle=True)
 for train_index, test_index in skf.split(train2, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gmm = GaussianMixture(n_components=4, init_params='kmeans', covariance_type='full', max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gmm.fit(np.concatenate([train3,test3],axis = 0))
prob = gmm.predict_proba(train3[test_index,:])
 prob_class = np.zeros(prob.shape[0])
 for j in range(prob.shape[0]):
if(np.argmax(prob,axis=1)[j] in {0, 1}):
prob_class[j] = np.max(prob,axis=1)[j]
else:
prob_class[j] = 1 - np.max(prob,axis=1)[j]
oof_gmm[idx1[test_index]] = prob_class
prob_test = gmm.predict_proba(test3)
 prob_test_class = np.zeros(prob_test.shape[0])
 for j in range(prob_test.shape[0]):
if(np.argmax(prob_test,axis=1)[j] in {0, 1}):
prob_test_class[j] = np.max(prob_test,axis=1)[j]
else:
prob_test_class[j] = 1 - np.max(prob_test,axis=1)[j]
preds_gmm[idx2] += prob_test_class / skf.n_splits
auc = roc_auc_score(train['target'], oof_gmm)
 print(f'AUC: {auc:.5}')
<feature_engineering>
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]
for itr in range(3):
test['target'] = preds_gmm
test.loc[test['target'] > 0.96, 'target'] = 1
test.loc[test['target'] < 0.14, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof_gmm > 0.98, 'target'] = 1
new_train.loc[oof_gmm < 0.02, 'target'] = 0
oof_gmm2 = np.zeros(len(train))
preds_gmm = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) , 
('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=2019,shuffle=True)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gmm = GaussianMixture(n_components=4, init_params='kmeans', covariance_type='full', max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gmm.fit(np.concatenate([train3,test3],axis = 0))
if len(oof_test_index)> 0:
prob = gmm.predict_proba(train3[oof_test_index,:])
 prob_class = np.zeros(prob.shape[0])
 for j in range(prob.shape[0]):
if(np.argmax(prob,axis=1)[j] in {0, 1}):
prob_class[j] = np.max(prob,axis=1)[j]
else:
prob_class[j] = 1 - np.max(prob,axis=1)[j]
oof_gmm2[idx1[test_index]] = prob_class
prob_test = gmm.predict_proba(test3)
 prob_test_class = np.zeros(prob_test.shape[0])
 for j in range(prob_test.shape[0]):
if(np.argmax(prob_test,axis=1)[j] in {0, 1}):
prob_test_class[j] = np.max(prob_test,axis=1)[j]
else:
prob_test_class[j] = 1 - np.max(prob_test,axis=1)[j]
preds_gmm[idx2] += prob_test_class / skf.n_splits
auc = roc_auc_score(train['target'], oof_gmm)
 print(f'AUC: {auc:.5}')
<save_to_csv>
train_set = train_set[train_set.GrLivArea < 8.25]
train_set = train_set[train_set.SalePrice < 13]
train_set = train_set[train_set.SalePrice > 10.75]
train_set.drop("Id", axis=1, inplace=True )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_gmm
sub.to_csv('submission.csv',index=False )<set_options>
dtrain = xgb.DMatrix(x_train, label = y)
 dtest = xgb.DMatrix(x_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100 )
warnings.filterwarnings("ignore")
<load_from_csv>
model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=2, learning_rate=0.1)
 model_xgb.fit(x_train, y )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
preds =(0.7*lasso_preds)+(0.3*xgb_preds )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<train_on_grid>
solution = pd.DataFrame({"id":test_set.Id, "SalePrice":preds})
 solution.to_csv("housePrice_solution.csv", index = False )