Kaggle competition: House Prices - Advanced Regression Techniques
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<choose_model_class>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
pd.options.display.max_rows = 100
class MyGM(GaussianMixture):
def __init__(self, n_components=1, covariance_type='full', tol=1e-3,
reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
weights_init=None, means_init=None, precisions_init=None,
random_state=0, warm_start=False,
verbose=0, verbose_interval=10, init_clusters=None, y=None, contamination=0.025, init_proba=None):
super().__init__(
 n_components=n_components, tol=tol, reg_covar=reg_covar,
max_iter=max_iter, n_init=n_init, init_params=init_params,
random_state=random_state, warm_start=warm_start,
verbose=verbose, verbose_interval=verbose_interval)
 self.init_clusters_ = np.asarray(init_clusters ).astype('int')
 self.y_ = y
self.contamination_ = contamination
self.init_proba_ = init_proba
def _initialize_parameters(self, X, random_state):

 n_samples, _ = X.shape
if self.init_params == 'kmeans':
resp = np.zeros(( n_samples, self.n_components))
label = cluster.KMeans(n_clusters=self.n_components, n_init=1,
random_state=random_state ).fit(X ).labels_
resp[np.arange(n_samples), label] = 1
elif self.init_params == 'random':
resp = random_state.rand(n_samples, self.n_components)
 resp /= resp.sum(axis=1)[:, np.newaxis]
elif self.init_params == 'clusters':
resp = np.zeros(( n_samples, self.n_components))
resp[np.arange(self.init_clusters_.shape[0]), self.init_clusters_] = 1
elif self.init_params == 'proba':
resp = self.init_proba_.copy()
resp[np.arange(self.init_clusters_.shape[0]), self.init_clusters_] = 1
resp /= resp.sum(axis=1)[:, np.newaxis]
else:
raise ValueError("Unimplemented initialization method '%s'"
% self.init_params)
 self._initialize(X, resp)
 def estimate_log_ratio(self, X):
weighted_log_prob = self._estimate_weighted_log_prob(X)
 return logsumexp(weighted_log_prob[:, 1::2], axis=1)- logsumexp(weighted_log_prob[:, 0::2], axis=1)
 def _estimate_log_prob(self, X):
base_pred = _estimate_log_gaussian_prob(
 X, self.means_, self.precisions_cholesky_, self.covariance_type)
 if self.y_ is None:
new_pred = base_pred
else:
new_pred = base_pred.copy()
for i in range(self.init_clusters_.shape[0]):
yi = self.init_clusters_[i]
for j in range(base_pred.shape[1]):
if(j - yi)% 2 == 0:
new_pred[i, j] += np.log(1 - self.contamination_)
 else:
new_pred[i, j] += np.log(self.contamination_)
 return new_pred 
<define_variables>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )
if test.shape[0] != 131073:
with single_threaded(np):
oof_preds = np.zeros(train.shape[0])
 test_preds = np.zeros(test.shape[0])
 num_models = 512
n_splits = 7
contamination = 0.025
n_cluster = 3
n_components = 2 * n_cluster
reg_covar=1.986808882
tol=0.012220889
n_random = 4
for RANDOM_STATE in tqdm_notebook(range(n_random)) :
for i in tqdm_notebook(range(num_models)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 n_features = train3.shape[1]
skf = StratifiedKFold(n_splits=n_splits, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
train_x_0 = train3[train_index][train2['target'].values[train_index] == 0]
train_x_1 = train3[train_index][train2['target'].values[train_index] == 1]
gm0 = GaussianMixture(n_components=n_cluster, weights_init=[1/n_cluster] * n_cluster,
n_init=5, random_state=RANDOM_STATE)
 _ = gm0.fit(train_x_0)
 cls0 = gm0.predict(train_x_0)
 ms0 = gm0.means_
ps0 = gm0.precisions_
gm1 = GaussianMixture(n_components=n_cluster,
n_init=5, reg_covar=reg_covar, tol=tol, random_state=RANDOM_STATE)
 _ = gm1.fit(train_x_1)
 cls1 = gm1.predict(train_x_1)
 ms1 = gm1.means_
ps1 = gm1.precisions_
train_cls = np.ones(train3[train_index].shape[0] ).astype(np.int)
 train_cls[train2['target'].values[train_index] == 0] = cls0 * 2
train_cls[train2['target'].values[train_index] == 1] = cls1 * 2 + 1
train_ms = np.zeros(( n_components, n_features))
train_ms[0::2] = ms0
train_ms[1::2] = ms1
train_ps = np.zeros(( n_components, n_features, n_features))
train_ps[0::2] = ps0
train_ps[1::2] = ps1
train_test3 = np.concatenate([train3[train_index], test3], axis=0)
 gm_all = MyGM(
 n_components=n_components, init_params='clusters', init_clusters=train_cls,
means_init=train_ms, precisions_init=train_ps,
weights_init=[1/n_components] * n_components, reg_covar=reg_covar, tol=tol, contamination=contamination, y=None, 
)
 _ = gm_all.fit(train_test3)
 gm_all.y_ = None
if True:
oof_pred6 = gm_all.estimate_log_ratio(train3[test_index])
 test_pred6 = gm_all.estimate_log_ratio(test3)
 else:
oof_pred6 = gm_all.predict_proba(train3[test_index])
 oof_pred6 = oof_pred6[:, 1::2].sum(axis=1)/ oof_pred6.sum(axis=1)
 test_pred6 = gm_all.predict_proba(test3)
 test_pred6 = test_pred6[:, 1::2].sum(axis=1)/ test_pred6.sum(axis=1)
 oof_preds[idx1[test_index]] += oof_pred6
test_preds[idx2] += test_pred6
oof_preds /= n_random
test_preds /= skf.n_splits * n_random
train_used =(train['wheezy-copper-turtle-magic'] < num_models)
 auc = roc_auc_score(train.loc[train_used, 'target'], oof_preds[train_used])
 print(f'AUC: {auc:.5}' )<compute_train_metric>
y_test = sample_submission.SalePrice
if test.shape[0] != 131073:
train_used =(train['wheezy-copper-turtle-magic'] < num_models)
 auc = roc_auc_score(train.loc[train_used, 'target'], expit(oof_preds[train_used]))
print(f'AUC: {auc:.5}' )<save_to_csv>
train_target = train.SalePrice
sub = pd.read_csv('.. /input/sample_submission.csv')
 if sub.shape[0] != 131073:
sub['target'] = expit(test_preds)
 sub.to_csv('submission.csv',index=False )<import_modules>
train = train.drop(['Id', 'SalePrice'], axis=1)
 test = test.drop(['Id'],axis=1 )
optuna.logging.disable_default_handler()
warnings.filterwarnings("ignore")
 %matplotlib inline
pd.set_option('max_columns', 9999)
 pd.set_option('max_rows', 9999)
 start = time.time()
os.listdir('.. /input/' )<load_from_csv>
data = pd.concat([train, test] )
def load_data(nrows=None, verbose=True):
train = pd.read_csv('.. /input/train.csv', nrows=nrows)
 test = pd.read_csv('.. /input/test.csv')
 if verbose:
print('Train Shape: {}'.format(train.shape))
print('Test Shape: {}'.format(test.shape))
return train, test<train_on_grid>
cat_feat = list(data.dtypes[data.dtypes == object].index )
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps<train_model>
data[cat_feat] = data[cat_feat].fillna('nan')
 num_feat = [f for f in data if f not in(cat_feat + ['Id', 'SalePrice'])]
def trainer(train, test, cv, params, model_type='qda', _scaler=True, SVD=None,
variance=False, _model=None, _plot=True, threshold=1.5):
_train, _test = train.copy() , test.copy()
_test['target'] = -1
cols = [col for col in _train.columns if not col in ['id', 'target', 'wheezy-copper-turtle-magic']]
preds = np.zeros(len(_test))
oof = np.zeros(len(_train))
oof_org = np.zeros(len(train_org))
for i in tqdm(range(512)) :
train_2 = _train[_train['wheezy-copper-turtle-magic'] == i]
test_2 = _test[_test['wheezy-copper-turtle-magic'] == i]
train_2_org = train_org[train_org['wheezy-copper-turtle-magic'] == i]
len_train = len(train_2)
 Y = train_2['target'].reset_index(drop=True ).values
train_index = train_2.index
test_index = test_2.index
train_org_index = train_2_org.index
merged_data = pd.concat([train_2, test_2], axis=0 ).reset_index(drop=True)[cols].values
if SVD is not None:
clf = TruncatedSVD(n_components=SVD)
 _X = clf.fit_transform(merged_data)
 X_org = clf.transform(train_2_org[cols].values)
 elif variance:
clf = VarianceThreshold(threshold=threshold)
 _X = clf.fit_transform(merged_data)
 X_org = clf.transform(train_2_org[cols].values)
 else:
_X = merged_data
X_org = clf.transform(train_2_org[cols].values)
 if _scaler:
_scaler = StandardScaler()
_X = _scaler.fit_transform(_X)
 X_org = _scaler.transform(X_org)
 X = _X[:len_train]
test_2 = _X[len_train:]
del merged_data, _X
gc.collect()
X = X.astype(np.float32)
 test_2 = test_2.astype(np.float32)
 X_org = X_org.astype(np.float32)
 for i,(trn_index, val_index)in enumerate(cv.split(X, Y)) :
if model_type == 'gmm':
ms, ps = get_mean_cov(X, Y)
 X_trn, X_val = X[trn_index], X[val_index]
Y_trn, Y_val = Y[trn_index], Y[val_index]
if model_type == 'nusvc':
model = NuSVC(**params)
 elif model_type == 'qda':
model = QuadraticDiscriminantAnalysis(**params)
 elif model_type == 'gmm':
model = GaussianMixture(**params, means_init=ms, precisions_init=ps)
 elif model_type == None:
model = _model
model.fit(X_trn, Y_trn)
 oof[train_index[val_index]] += model.predict_proba(X_val)[:,1]
preds[test_index] += model.predict_proba(test_2)[:,1] / cv.n_splits
oof_org[train_org_index] += model.predict_proba(X_org)[:,1] / cv.n_splits
if model_type == 'gmm':
oof = 1.0 - oof
preds = 1.0 - preds
oof_org = 1.0 - oof_org
auc = roc_auc_score(_train['target'], oof[:_train.shape[0]])
 auc_org = roc_auc_score(train_org['target'], oof_org)
 if _plot:
fig = plt.figure(figsize=(6, 4))
sns.distplot(np.array(preds), kde=False, bins=25)
 plt.title('Model [{}] AUC: {:.5f}'.format(model_type, auc_org))
plt.show()
return np.array(preds), oof[:train.shape[0]]<split>
cat_nunique = data[cat_feat].nunique()
print(cat_nunique)
 cat_feat = list(cat_nunique[cat_nunique < 30].index )
train, test = load_data()
train_org = train.copy()<concatenate>
dummy_data = pd.get_dummies(data[cat_feat], columns=cat_feat)
 dummy_cols = list(set(dummy_data))
dummy_data = dummy_data[dummy_cols]
data = pd.concat([data[num_feat].fillna(-999),
dummy_data], axis=1)
def PsuedoLabeling(train, test, reg_param, _min=0.01, _max=0.99):
_train = train.copy()
_test = test.copy()
qda_params = {
'reg_param': reg_param
}
cv = StratifiedKFold(n_splits=15)
 preds, oof = trainer(_train, _test, cv, qda_params, model_type='qda', variance=True, _plot=False, _scaler=False)
 _test['target'] = preds
test_0 = _test[_test['target'] < _min]
test_0['target'] = 0
test_1 = _test[_test['target'] > _max]
test_1['target'] = 1
train = pd.concat([train, test_0, test_1], axis=0 ).reset_index(drop=True)
 return train
def run_PsuedoLabeling(train, test, reg_param, _min=0.01, _max=0.99, _iter=20):
reg_param = random.uniform(0.1, 0.6)
 for _ in range(_iter):
before_train_shape = train.shape[0]
train = PsuedoLabeling(train, test, reg_param, _min=_min, _max=_max)
 train = train.drop_duplicates(keep='first' ).reset_index(drop=True)
 print('Train Data Shape: {}'.format(train.shape))
if train.shape[0] ==before_train_shape:
break
return train<prepare_x_and_y>
train = data.iloc[:train.shape[0], :]
test = data.iloc[train.shape[0]:, :]
def change_label(train, test, reg_param):
modified_train = train.copy()
_train = train.copy()
_test = test.copy()
qda_params = {
'reg_param': reg_param
}
cv = StratifiedKFold(n_splits=15, random_state=46)
 preds, oof = trainer(_train, _test, cv, qda_params, model_type='qda', variance=True, _plot=False, _scaler=False)
 df = pd.DataFrame({
'id': train['id'].values,
'target': train['target'].values,
'oof': oof
})
 df_0 = df[(df['target'] == 1)&(df['oof'] < 0.01)]
df_1 = df[(df['target'] == 0)&(df['oof'] > 0.99)]
modified_train.loc[df_0.index, 'target'] = 0
modified_train.loc[df_1.index, 'target'] = 1
modified_rows = len(df_0)+ len(df_1)
 print('Modified {} rows'.format(modified_rows))
return modified_train, modified_rows
def run_change_label(train, test, reg_param, _iter=20):
reg_param = random.uniform(0.1, 0.6)
 for i in range(_iter):
train, modified_rows = change_label(train, test, reg_param)
 train.reset_index(drop=True, inplace=True)
 if modified_rows < 10:
break
return train<save_to_csv>
X_train = train
y_train = train_target
X_test = test
y_test = sample_submission.SalePrice
%%time
prep_start = time.time()
train = run_PsuedoLabeling(train, test, reg_param=0.5, _min=0.04, _max=0.96, _iter=10)
 train = run_change_label(train, test, reg_param=0.5, _iter=3)
 train.to_csv('Preprocessed_train.csv' )<train_model>
params = {'n_estimators': 130,
'learning_rate': 0.01,
'max_depth': 3,
'min_child_weight': 1,
'subsample': 1,
'colsample_bytree': 1,
'objective': 'reg:linear',
'n_jobs': 4}
clf_xgb = xgb.XGBRegressor(**params)
 clf_xgb.fit(X_train, y_train, eval_metric='mae', eval_set=[(X_train, y_train),(X_test, y_test)] )
prep_end = time.time()
elapsedtime = prep_end - prep_start
s = datetime.timedelta(seconds=elapsedtime)
 print('Preprocessing 1 epoch time: {}'.format(str(s)))
 print('Final Train Data Shape: {}'.format(train.shape))<create_dataframe>
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials
tr = pd.DataFrame({
'target': train['target'].values
})
 te = pd.DataFrame()<train_on_grid>
def score(params):
params['max_depth'] = int(params['max_depth'])
 params['n_jobs'] = -1
print("Training with params : ", params)
 clf = xgb.XGBRegressor(**params)
 clf.fit(X_train, y_train)
 y_pred_xgb_test = clf.predict(X_test)
 mae = mean_absolute_error(y_test, y_pred_xgb_test)
 result = {'loss': mae, 'status': STATUS_OK}
print('TEST ROC AUC: {0:.4f}'.format(mae))
return result
space = {
'max_depth' : hp.choice('max_depth', range(1, 15)) ,
'n_estimators': hp.choice('n_estimators', range(100, 1000)) ,
'eta': hp.quniform('eta', 0.025, 0.5, 0.10),
'max_depth': hp.choice('max_depth', np.arange(1, 14, dtype=int)) ,
'min_child_weight': hp.quniform('min_child_weight', 1, 6, 0.5),
'subsample': hp.quniform('subsample', 0.5, 1, 0.05),
'gamma': hp.quniform('gamma', 0.5, 1, 0.05),
'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05),
'eval_metric': 'mae',
'objective': 'reg:linear',
'nthread': 4,
'booster': 'gbtree',
'tree_method': 'exact',
'silent': 1
}
trials = Trials()
best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=20 )
%%time
name = 'qda'
qda_params = {
'reg_param': 0.01849,
'priors': 17.0978,
'tol': 6.943e-06
}
cv = StratifiedKFold(n_splits=15, random_state=46, shuffle=True)
 preds, oof = trainer(train, test, cv, qda_params, model_type=name.split('_')[0],
variance=True, _scaler=True, threshold=2.99)
 tr[name] = oof
te[name] = preds<train_on_grid>
trials.best_trial
%%time
name = 'nusvc'
nusvc_params = {
'probability': True,
'kernel': 'poly',
'degree': 3,
'gamma': 'auto',
'coef0': 0.159,
'nu': 0.1486,
'tol': 1.8909e-4
}
cv = StratifiedKFold(n_splits=5, random_state=46)
 preds, oof = trainer(train, test, cv, nusvc_params, model_type=name.split('_')[0], SVD=20)
 tr[name] = oof
te[name] = preds<train_on_grid>
clf_xgb = xgb.XGBRegressor(**best)
 clf_xgb.fit(X_train, y_train, eval_metric='mae', eval_set=[(X_train, y_train),(X_test, y_test)] )
%%time
name = 'gmm'
gmm_params = {
'n_components': 2,
'init_params': 'kmeans',
'covariance_type': 'full',
'tol': 0.001,
'reg_covar': 0.000389422,
'max_iter': 536,
'n_init': 1
}
cv = StratifiedKFold(n_splits=5, random_state=46, shuffle=True)
 preds, oof = trainer(train, test, cv, gmm_params, model_type=name.split('_')[0],
variance=True, _scaler=False, threshold=2.21)
 tr[name] = oof
te[name] = preds<feature_engineering>
predict = clf_xgb.predict(X_test )
for col in te.columns:
tr_rank = tr[col].rank(ascending=True)
 te_rank = te[col].rank(ascending=True)
 tr['{}_rank'.format(col)] = tr_rank.apply(lambda x:(x - tr_rank.min())/(tr_rank.max() - tr_rank.min()))
te['{}_rank'.format(col)] = te_rank.apply(lambda x:(x - te_rank.min())/(te_rank.max() - te_rank.min()))<normalization>
submission = pd.DataFrame({
"Id": sample_submission["Id"],
"SalePrice": predict
})
 submission.to_csv('submission.csv', index=False )