Kaggle competition: House Prices - Advanced Regression Techniques
oof_qda1 = np.zeros(len(train))
preds_qda1 = np.zeros(len(test))<define_search_space>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
params = np.arange(0.1, 0.5, 0.1)
 parameters = [{'reg_param': params}]
reg_params = np.zeros(512 )<drop_column>
train_ID = train['Id']
test_ID = test['Id']
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=2)) , 
('sscaler', StandardScaler()),
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
qda = QuadraticDiscriminantAnalysis()
clf = GridSearchCV(qda, parameters, cv=4)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 reg_params[i] = clf.best_params_['reg_param']
oof_qda1[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_qda1[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits<compute_test_metric>
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
print('QDA with VarianceThreshold and StandardScaler CV Score =',round(roc_auc_score(train['target'], oof_qda1), 5))<save_to_csv>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )
sub['target'] = preds_qda1
sub['target'].to_csv('submission_qda1.csv', index=False )<prepare_output>
train["SalePrice"] = np.log1p(train["SalePrice"] )
sub['target'] = preds_qda1
sub['target'].hist(bins=100, alpha=0.6 )<define_variables>
ntrain = train.shape[0]
y_train = train.SalePrice.values
concat_data = pd.concat(( train, test)).reset_index(drop=True)
 concat_data.drop(['SalePrice'], axis=1, inplace=True )
oof_qda2 = np.zeros(len(train))
preds_qda2 = np.zeros(len(test))<define_variables>
concat_data.isnull().sum().sort_values(ascending=False )
dict = dict()<normalization>
concat_data["PoolQC"] = concat_data["PoolQC"].fillna("None")
 concat_data["MiscFeature"] = concat_data["MiscFeature"].fillna("None")
 concat_data["Alley"] = concat_data["Alley"].fillna("None")
 concat_data["Fence"] = concat_data["Fence"].fillna("None")
 concat_data["FireplaceQu"] = concat_data["FireplaceQu"].fillna("None" )
%%time
for s in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==s]
test2 = test[test['wheezy-copper-turtle-magic']==s]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 dict[s] = train3.shape[1]<drop_column>
concat_data["LotFrontage"] = concat_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('scaler', StandardScaler()), 
('fa', FeatureAgglomeration(n_clusters=dict[i]))
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_qda2[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_qda2[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits<compute_test_metric>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
concat_data[col] = concat_data[col].fillna('None' )
print('QDA with PCA CV Score =',round(roc_auc_score(train['target'], oof_qda2), 5))<save_to_csv>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
concat_data[col] = concat_data[col].fillna(0 )
sub['target'] = preds_qda2
sub.to_csv('submission_qda2.csv', index=False )<prepare_output>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
concat_data[col] = concat_data[col].fillna(0 )
sub['target'] = preds_qda2
sub['target'].hist(bins=100, alpha=0.6 )<define_variables>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
concat_data[col] = concat_data[col].fillna('None' )
oof_gmm = np.zeros(len(train))
preds_gmm = np.zeros(len(test))<data_type_conversions>
concat_data["MasVnrType"] = concat_data["MasVnrType"].fillna("None")
 concat_data["MasVnrArea"] = concat_data["MasVnrArea"].fillna(0 )
def get_mean_cov(x, y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms, ps<create_dataframe>
concat_data['MSZoning'] = concat_data['MSZoning'].fillna(concat_data['MSZoning'].mode() [0] )
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5))
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:], train2.loc[train_index]['target'].values)
 clf = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001, reg_covar=0.001, max_iter=100, n_init=1, means_init=ms, precisions_init=ps)
 clf.fit(np.concatenate([train3,test3],axis = 0))
oof_gmm[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,0]
preds_gmm[idx2] += clf.predict_proba(test3)[:,0] / skf.n_splits<compute_test_metric>
concat_data = concat_data.drop(['Utilities'], axis=1 )
print('GaussianMixture with VarianceThreshold CV Score =',round(roc_auc_score(train['target'], oof_gmm), 5))<save_to_csv>
concat_data["Functional"] = concat_data["Functional"].fillna("Typ" )
sub['target'] = preds_gmm
sub.to_csv('submission_gmm.csv', index=False )<define_variables>
concat_data['KitchenQual'] = concat_data['KitchenQual'].fillna(concat_data['KitchenQual'].mode() [0] )
oof_lr = np.zeros(len(train))
preds_lr = np.zeros(len(test))<create_dataframe>
concat_data['Exterior1st'] = concat_data['Exterior1st'].fillna(concat_data['Exterior1st'].mode() [0])
 concat_data['Exterior2nd'] = concat_data['Exterior2nd'].fillna(concat_data['Exterior2nd'].mode() [0] )
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('poly', PolynomialFeatures(degree=2)) , 
('sc', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = LogisticRegression(solver='saga', penalty='l2', C=0.01, tol=0.001, n_jobs=-1)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_lr[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_lr[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits<compute_test_metric>
concat_data['SaleType'] = concat_data['SaleType'].fillna(concat_data['SaleType'].mode() [0] )
print('Logistic Regression with VarianceThreshold CV Score =',round(roc_auc_score(train['target'], oof_lr), 5))<save_to_csv>
concat_data['MSSubClass'] = concat_data['MSSubClass'].fillna("None" )
sub['target'] = preds_lr
sub.to_csv('submission_lr.csv', index=False )<define_variables>
concat_data['MSSubClass'] = concat_data['MSSubClass'].apply(str)
 concat_data['OverallCond'] = concat_data['OverallCond'].astype(str)
 concat_data['YrSold'] = concat_data['YrSold'].astype(str)
 concat_data['MoSold'] = concat_data['MoSold'].astype(str )
oof_ls = np.zeros(len(train))
preds_ls = np.zeros(len(test))<create_dataframe>
CatFeatures =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 'ExterQual', 'ExterCond','HeatingQC',
'PoolQC', 'KitchenQual', 'BsmtFinType1', 'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure',
'GarageFinish', 'LandSlope', 'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass',
'OverallCond', 'YrSold', 'MoSold')
 lbl = LabelEncoder()
for col in CatFeatures:
lbl.fit(list(concat_data[col].values))
concat_data[col] = lbl.transform(list(concat_data[col].values))
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5))
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = LabelSpreading(gamma=0.01, kernel='rbf', max_iter=10)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_ls[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_ls[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits<compute_test_metric>
concat_data['TotalSF'] = concat_data['TotalBsmtSF'] + concat_data['1stFlrSF'] + concat_data['2ndFlrSF']
print('LabelSpreading with VarianceThreshold CV Score =',round(roc_auc_score(train['target'], oof_ls), 5))<save_to_csv>
concat_data = pd.get_dummies(concat_data)
 print(concat_data.shape )
sub['target'] = preds_ls
sub.to_csv('submission_ls.csv', index=False )<define_variables>
train = concat_data[:ntrain]
test = concat_data[ntrain:]
oof_knn = np.zeros(len(train))
preds_knn = np.zeros(len(test))<create_dataframe>
import xgboost as xgb
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=2)) , 
('scaler', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = KNeighborsClassifier(n_neighbors=17, p=2.9, n_jobs=-1)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_knn[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_knn[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits<compute_test_metric>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.5, gamma=0.05,
learning_rate=0.05, max_depth=3,
min_child_weight=2, n_estimators=2000,
reg_alpha=0.5, reg_lambda=0.8,
subsample=0.5, random_state =7 )
print('kNN with VarianceThreshold and StandardScaler CV Score =',round(roc_auc_score(train['target'], oof_knn), 5))<save_to_csv>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
sub['target'] = preds_knn
sub.to_csv('submission_knn.csv', index=False )<prepare_output>
print(np.sqrt(mean_squared_error(y_train, xgb_train_pred)) )
sub['target'] = preds_knn
sub['target'].hist(bins=100, alpha=0.6 )<define_variables>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = xgb_pred
sub.to_csv('submission.csv',index=False )