Kaggle competition: House Prices - Advanced Regression Techniques
from sklearn.svm import NuSVC<normalization>
print('Python: {}'.format(sys.version))
print('scipy: {}'.format(scipy.__version__))
print('numpy: {}'.format(np.__version__))
print('matplotlib: {}'.format(matplotlib.__version__))
print('pandas: {}'.format(pd.__version__))
print('sklearn: {}'.format(sklearn.__version__))
print('seaborn: {}'.format(sns.__version__))
for i in tqdm_notebook(range(512)) :
trn2 = trn_new[trn_new['wheezy-copper-turtle-magic']==i]
tst2 = tst[tst['wheezy-copper-turtle-magic']==i]
idx1 = trn2.index; idx2 = tst2.index
trn2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(trn2[cols]), pd.DataFrame(tst2[cols])])
 usecols = [var for var in cols if data[var].std() >2]
data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 trn3 = data2[:trn2.shape[0]]; tst3 = data2[trn2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 oof_tmp_qda = np.zeros(trn2.shape[0])
 oof_tmp_nusvc = np.zeros(trn2.shape[0])
 for trn_index, tst_index in skf.split(trn2, trn2['target']):
clf_qda = QuadraticDiscriminantAnalysis(0.6)
 clf_qda.fit(trn3[trn_index,:],trn2.loc[trn_index]['target'])
 oof_qda[idx1[tst_index]] = clf_qda.predict_proba(trn3[tst_index,:])[:,1]
oof_tmp_qda[tst_index] = clf_qda.predict_proba(trn3[tst_index,:])[:,1]
preds_qda[idx2] += clf_qda.predict_proba(tst3)[:,1] / skf.n_splits
clf_nusvc = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=4, nu=0.5, coef0=0.053)
 clf_nusvc.fit(trn3[trn_index,:],trn2.loc[trn_index]['target'])
 oof_nusvc[idx1[tst_index]] = clf_nusvc.predict_proba(trn3[tst_index,:])[:,1]
oof_tmp_nusvc[tst_index] = clf_nusvc.predict_proba(trn3[tst_index,:])[:,1]
preds_nusvc[idx2] += clf_nusvc.predict_proba(tst3)[:,1] / skf.n_splits
trn2['pred_qda'] = oof_tmp_qda
trn2['pred_nusvc'] = oof_tmp_nusvc
auc_bench_qda = roc_auc_score(trn2.loc[trn2['target_old'].notnull() ]['target_old'], trn2.loc[trn2['target_old'].notnull() ]['pred_qda'])
 auc_basic_qda = auc_bench_qda
auc_bench_nusvc = roc_auc_score(trn2.loc[trn2['target_old'].notnull() ]['target_old'], trn2.loc[trn2['target_old'].notnull() ]['pred_nusvc'])
 auc_basic_nusvc = auc_bench_nusvc
best_com = 0
for loop in range(6):
components = out_info[i][1].shape[0]
trn2['tgt'] = np.nan
gmm1 = GaussianMixture(n_components=components,max_iter = max_iter)
 trn2.loc[trn2['target']==1,'tgt'] = gmm1.fit_predict(trn2[trn2['target']==1][usecols].values)+components
gmm2 = GaussianMixture(n_components=components,max_iter = max_iter)
 trn2.loc[trn2['target']==0,'tgt'] = gmm2.fit_predict(trn2[trn2['target']==0][usecols].values)
 trn3 = trn2[usecols].values
if add_sample:
tmp = gmm1.sample(n_samples = sample_size)
 trn2_pos = pd.DataFrame(tmp[0],columns=usecols)
 trn2_pos['tgt'] = tmp[1]+components
trn2_pos['target'] = np.nan
tmp = gmm2.sample(n_samples = sample_size)
 trn2_neg = pd.DataFrame(tmp[0],columns=usecols)
 trn2_neg['tgt'] = tmp[1]
trn2_neg['target'] = np.nan
trn2_add = pd.concat([trn2,trn2_pos,trn2_neg] ).sample(frac=1 ).reset_index(drop=True)
 else:
trn2_add = trn2.copy()
trn3 = trn2_add[usecols].values
skf = StratifiedKFold(n_splits=11, random_state=42)
 oof_tmp_qda = np.zeros(trn2_add.shape[0])
 preds_tmp_qda = np.zeros(tst3.shape[0])
 oof_tmp_nusvc = np.zeros(trn2_add.shape[0])
 preds_tmp_nusvc = np.zeros(tst3.shape[0])
 for trn_index, tst_index in skf.split(trn2_add, trn2_add['tgt']):
clf_qda = QuadraticDiscriminantAnalysis(0.6)
 clf_nusvc = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=4, nu=0.5, coef0=0.053)
 try:
clf_qda.fit(trn3[trn_index,:],trn2_add.loc[trn_index]['tgt'])
 clf_nusvc.fit(trn3[trn_index,:],trn2_add.loc[trn_index]['tgt'])
 except:
break
oof_tmp_qda[tst_index] = clf_qda.predict_proba(trn3[tst_index,:])[:,components:].sum(axis=1)
 preds_tmp_qda += clf_qda.predict_proba(tst3)[:,components:].sum(axis=1)/ skf.n_splits
oof_tmp_nusvc[tst_index] = clf_nusvc.predict_proba(trn3[tst_index,:])[:,components:].sum(axis=1)
 preds_tmp_nusvc += clf_nusvc.predict_proba(tst3)[:,components:].sum(axis=1)/ skf.n_splits
trn2_add['pred_qda'] = oof_tmp_qda
trn2_add['pred_nusvc'] = oof_tmp_nusvc
auc_tmp_qda = roc_auc_score(trn2_add[trn2_add['target_old'].notnull() ]['target_old'],trn2_add[trn2_add['target_old'].notnull() ]['pred_qda'])
 auc_tmp_nusvc = roc_auc_score(trn2_add[trn2_add['target_old'].notnull() ]['target_old'],trn2_add[trn2_add['target_old'].notnull() ]['pred_nusvc'])
 if auc_bench_qda < auc_tmp_qda:
del trn2['pred_qda']
auc_bench_qda = auc_tmp_qda
trn2 = trn2.merge(trn2_add[['id','pred_qda']],'left','id')
 preds_qda[idx2] = preds_tmp_qda
oof_qda[idx1] = trn2['pred_qda'].values
if auc_bench_nusvc < auc_tmp_nusvc:
del trn2['pred_nusvc']
auc_bench_nusvc = auc_tmp_nusvc
trn2 = trn2.merge(trn2_add[['id','pred_nusvc']],'left','id')
 preds_nusvc[idx2] = preds_tmp_nusvc
oof_nusvc[idx1] = trn2['pred_nusvc'].values
print("第{}轮QDA的CV为：{}，basic的CV为：{}".format(i,auc_bench_qda,auc_basic_qda))
print("第{}轮NuSVC的CV为：{}，basic的CV为：{}".format(i,auc_bench_nusvc,auc_basic_nusvc))
trn_new['pred_qda'] = oof_qda
trn_new['pred_nusvc'] = oof_nusvc<compute_test_metric>
%matplotlib inline
auc_qda = roc_auc_score(trn_new.loc[trn_new['target_old'].notnull() ]['target_old'], trn_new.loc[trn_new['target_old'].notnull() ]['pred_qda'])
 auc_nusvc = roc_auc_score(trn_new.loc[trn_new['target_old'].notnull() ]['target_old'], trn_new.loc[trn_new['target_old'].notnull() ]['pred_nusvc'] )<feature_engineering>
df_train=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
def find_the_threshold2(n,m,oof_qda,oof_nusvc,oof2,a,b):
for i in range(oof_qda.shape[0]):
if oof_qda[i]>n:
oof2[i] = oof_qda[i]
elif oof_qda[i]<m:
oof2[i] = oof_qda[i]
else:
oof2[i] = oof_nusvc[i]*a+oof_qda[i]*b
return oof2<compute_train_metric>
df_train.isnull().sum().sum() , df_test.isnull().sum().sum()
oof2 = np.zeros(len(trn))
print(roc_auc_score(trn_new.loc[trn_new['target_old'].notnull() ]['target_old'],find_the_threshold2(0.99997,0.00007,
trn_new.loc[trn_new['target_old'].notnull() ]['pred_qda'],
trn_new.loc[trn_new['target_old'].notnull() ]['pred_nusvc'],
oof2,
0.5,0.5)) )<feature_engineering>
all_features = pd.concat(( df_train.iloc[:, 1:-1], df_test.iloc[:, 1:]))
for i in range(preds_qda.shape[0]):
if preds_qda[i]>0.99997:
preds[i] = preds_qda[i]
elif preds_qda[i]<0.00007:
preds[i] = preds_qda[i]
else:
preds[i] = preds_nusvc[i]*0.5+preds_qda[i]*0.5<save_to_csv>
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
all_features[numeric_features] = all_features[numeric_features].apply(
 lambda x:(x - x.mean())/(x.std()))
all_features[numeric_features] = all_features[numeric_features].fillna(0 )
sub['target'] = preds
sub.to_csv('submission.csv',index=False )<import_modules>
all_features = pd.get_dummies(all_features, dummy_na=True)
 all_features.shape
%matplotlib inline<load_from_csv>
df_train.isnull().sum().sum() , df_test.isnull().sum().sum()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<train_on_grid>
n_train = df_train.shape[0]
train_features = np.array(all_features[:n_train].values, dtype=np.float32)
 test_features = np.array(all_features[n_train:].values, dtype=np.float32)
 train_labels = np.array(df_train.SalePrice.values.reshape(-1, 1), dtype=np.float32 )
def get_kmeans_clusters(x, y, k_pos=1, k_neg=1):
x_zeros = x[y==0]
x_ones = x[y==1]
model_0 = KMeans(n_clusters=k_neg)
 model_1 = KMeans(n_clusters=k_pos)
 model_0.fit(x_zeros)
 model_1.fit(x_ones)
 model_0_clus = [x_zeros[model_0.labels_==k] for k in range(model_0.n_clusters)]
model_1_clus = [x_ones[model_1.labels_==k] for k in range(model_1.n_clusters)]
return model_1_clus + model_0_clus
def fit_multicluster_gmm(x, y, xt, k_pos, k_neg, max_iter=100):

 clusters = get_kmeans_clusters(x, y, k_pos=k_pos, k_neg=k_neg)
 for i in range(len(clusters)) :
x_cluster = clusters[i]
model = ShrunkCovariance()
model.fit(x_cluster)
 if(i==0):
ps = np.expand_dims(model.precision_, axis=0)
 ms = np.expand_dims(model.location_, axis=0)
 else:
ps = np.concatenate([ps, np.expand_dims(model.precision_, axis=0)], axis=0)
 ms = np.concatenate([ms, np.expand_dims(model.location_, axis=0)], axis=0)
 gm = mixture.GaussianMixture(n_components=k_pos+k_neg,
init_params='random',
covariance_type='full',
tol=0.001,
reg_covar=0.001,
max_iter=100,
n_init=5,
means_init=ms,
precisions_init=ps)
 gm.fit(np.vstack(( x.astype(np.float), xt.astype(np.float))))
preds = gm.predict_proba(x.astype(np.float)) [:,0]
score = roc_auc_score(y, preds)
 return score, gm, k_pos, k_neg
def get_mean_cov(x,y, model=GraphicalLasso() , max_iter=100):
try:
model.set_params(**{'max_iter':200})
 except:
pass
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps<prepare_x_and_y>
y = train_labels
X = train_features
X_test = test_features
def extract_wheezy_copper_turtle_magic(train, i):
train2 = train[train['wheezy-copper-turtle-magic']==i].copy()
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
target = train2['target'].astype(np.int ).values
train2.reset_index(drop=True, inplace=True)
 return train2.drop(['id', 'target'], axis=1), test2.drop(['id'], axis=1), idx1, idx2, target<concatenate>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1 )
def data_augmentation(X, y, gm_means, gm_dist):
for clus in range(len(gm_means)) :
A = 2*gm_means[clus,].reshape(1,-1)- X[gm_dist==clus]
B = X[gm_dist==clus]
t = y[gm_dist==clus]
if clus==0:
Xn = np.vstack(( A, B))
yn = np.concatenate(( t,t))
else:
Xn = np.vstack(( Xn, A, B))
yn = np.concatenate(( yn, t,t))
return Xn, yn 
<define_variables>
kfold = KFold(n_splits=20, random_state= 0, shuffle = True )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
test_preds = np.zeros(len(test))
oof_gmm = np.zeros(len(train))
test_preds_gmm = np.zeros(len(test))
oof_gmm_2 = np.zeros(len(train))
test_preds_gmm_2 = np.zeros(len(test))
trials = 3
cat_dict = dict()
cluster_report = list()
for i in tqdm(range(512)) :
train2, test2, idx1, idx2, target = extract_wheezy_copper_turtle_magic(train, i)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ])
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 cat_dict[i] = train3.shape[1]
svd = PCA(n_components=cat_dict[i])
 svd.fit(np.vstack(( train3, test3)))
 train3 = svd.transform(train3)
 test3 = svd.transform(test3)
 try:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=3, k_neg=3)
 except:
try:
print("Falling back to k_pos/k_neg=2")
 score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=2, k_neg=2)
 except:
print("Falling back to k_pos/k_neg=1")
 score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=1)
 clusters = gm.predict_proba(train3 ).shape[1]
oof_gmm[idx1] = np.sum(gm.predict_proba(train3)[:,:clusters//2], axis=1)
 test_preds_gmm[idx2] += np.sum(gm.predict_proba(test3)[:,:clusters//2], axis=1)
 try:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=3)
 except:
try:
print("Falling back to k_pos/k_neg=2")
 score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=2)
 except:
print("Falling back to k_pos/k_neg=1")
 score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=1)
 clusters = gm.predict_proba(train3 ).shape[1]
oof_gmm_2[idx1] = gm.predict_proba(train3)[:,0]
test_preds_gmm_2[idx2] += gm.predict_proba(test3)[:,0]<compute_test_metric>
clf = Lasso(alpha=102)
 clf.fit(train_X, train_y)
 y_pred = clf.predict(val_X)
 y_train_pred = clf.predict(train_X )
oof_auc_gmm = roc_auc_score(train['target'], oof_gmm)
 print('OOF AUC: =',round(oof_auc_gmm, 5))
oof_auc_gmm_2 = roc_auc_score(train['target'], oof_gmm_2)
 print('OOF AUC: =',round(oof_auc_gmm_2, 5))
oof_auc_blend = roc_auc_score(train['target'],(oof_gmm+oof_gmm_2)/2)
 print('OOF AUC: =',round(oof_auc_blend, 5))<save_to_csv>
print("Train acc: " , r2_score(train_y, y_train_pred))
print("Test acc: ", r2_score(val_y, y_pred))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = test_preds_gmm
sub.to_csv('submission_gmm.csv',index=False)
 sub['target'] = test_preds_gmm_2
sub.to_csv('submission_gmm_2.csv',index=False)
 sub['target'] =(test_preds_gmm + test_preds_gmm_2)/2
sub.to_csv('submission_blend.csv',index=False )<load_from_csv>
print("Train acc: " , clf.score(train_X, train_y))
print("Test acc: ", clf.score(val_X, val_y))
train_df = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<train_model>
final_labels = clf.predict(X_test )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models, weights = None):
self.models = models
self.weights = weights
assert len(self.models)== len(self.weights),('Len models != len weights')
 assert abs(np.sum(self.weights)- 1)< 0.01,('weight sum != 1')
 def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict_proba(X)[:,1] for model in self.models_
])
 if self.weights:
return np.sum(predictions * self.weights, axis=1)
 else:
return np.mean(predictions, axis=1 )<predict_on_test>
final_result = pd.DataFrame({'Id': df_test['Id'], 'SalePrice': final_labels} )
def get_predicts_with_validate(model, X, Y, n_folds = 3, X_test=[]):
kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 42)
 scores = list()
out_of_fold_predictions = np.zeros(( X.shape[0]))
predicts_list = list()
for train_index, test_index in kf.split(X,Y):
X_train, X_valid = X.iloc[train_index], X.iloc[test_index]
Y_train, Y_valid = Y.iloc[train_index], Y.iloc[test_index]
model.fit(X_train,Y_train)
 predicts = model.predict(X_valid)
 out_of_fold_predictions[test_index] = predicts
scores.append(roc_auc_score(Y_valid, predicts))
if len(X_test)>0:
test_predicts = model.predict(X_test)
 predicts_list.append(test_predicts)
 return out_of_fold_predictions, scores, np.mean(predicts_list, axis=0 )<train_on_grid>
final_result.to_csv('submission_lasso.csv', index=False)
 print("Your submission was successfully saved!" )
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
try:
kmeans_ = KMeans(3)
 clusters = kmeans_.fit_predict(x2)
 p1_list = list()
m1_list = list()
for i in range(3):
model.fit(x2[np.where(clusters == i)[0]])
 p1_list.append(model.precision_)
 m1_list.append(model.location_)
 onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
kmeans_ = KMeans(3)
 clusters = kmeans_.fit_predict(x2b)
 for i in range(3):
model.fit(x2b[np.where(clusters == i)[0]])
 p1_list.append(model.precision_)
 m1_list.append(model.location_)
 n_clusters = 3
except:
print('!!!')
 kmeans_ = KMeans(2)
 clusters = kmeans_.fit_predict(x2)
 p1_list = list()
m1_list = list()
for i in range(2):
model.fit(x2[np.where(clusters == i)[0]])
 p1_list.append(model.precision_)
 m1_list.append(model.location_)
 onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
kmeans_ = KMeans(2)
 clusters = kmeans_.fit_predict(x2b)
 for i in range(2):
model.fit(x2b[np.where(clusters == i)[0]])
 p1_list.append(model.precision_)
 m1_list.append(model.location_)
 n_clusters = 2
ms = np.stack(m1_list)
 ps = np.stack(p1_list)
 return ms,ps, n_clusters<choose_model_class>
submission_lasso = pd.read_csv("submission_lasso.csv")
 submission_lasso
QDA = QuadraticDiscriminantAnalysis(reg_param=0.5 )<normalization>
reg = LinearRegression()
reg.fit(train_X,train_y)
 reg.score(val_X,val_y )
cols = [c for c in train_df.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 preds = np.zeros(len(train_df))
for i in tqdm(range(512)) :
train2 = train_df[train_df['wheezy-copper-turtle-magic']==i]
idx1 = train2.index;
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 models = [QDA]
averaged_models = AveragingModels(models=models, weights=[1])
 out_of_fold_predictions, scores,_ = get_predicts_with_validate(averaged_models, pd.DataFrame(train3), train2['target'], 11)
 preds[idx1] = out_of_fold_predictions
auc = roc_auc_score(train_df['target'], preds)
 print('QDA scores CV =',round(auc,5))<feature_engineering>
clf = ensemble.GradientBoostingRegressor(n_estimators = 400, max_depth = 5, min_samples_split = 2,
learning_rate = 0.1, loss = 'ls')
 clf.fit(train_X, train_y)
 clf.score(val_X,val_y )
train_df['preds'] = preds
print(len(train_df[(( train_df['target']==0)&(train_df['preds']>=0.95)) |(( train_df['target']==1)&(train_df['preds']<=0.05)) ]))
train_df = train_df[(( train_df['target']==0)&(train_df['preds']<=0.95)) |(( train_df['target']==1)&(train_df['preds']>=0.05)) ]
train_df.drop(['preds'], axis=1, inplace=True)
 train_df.reset_index(inplace=True,drop=True )<define_variables>
y_pred = reg.predict(X_test)
 y_pred
cols = [c for c in train_df.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 preds = np.zeros(len(train_df))
test_preds = np.zeros(len(test))
preds_gm = np.zeros(len(train_df))
test_preds_gm = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train_df[train_df['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 models = [QDA]
averaged_models = AveragingModels(models=models, weights=[1])
 out_of_fold_predictions, scores, test_predicts = get_predicts_with_validate(averaged_models, pd.DataFrame(train3), train2['target'], 11, test3)
 preds[idx1] = out_of_fold_predictions
test_preds[idx2] = test_predicts
preds = preds
test_preds = test_preds
auc = roc_auc_score(train_df['target'], preds)
 print('QDA scores CV =',round(auc,5))<feature_engineering>
final_labels1 = clf.predict(X_test)
 final_labels1
test['target'] = test_preds
print(len(test[(test['target']<=0.01)|(test['target']>=0.99)]))<define_variables>
final_labels1 = final_labels1.flatten()
preds = np.zeros(len(train_df))
test_preds_ = np.zeros(len(test))
preds_gm = np.zeros(len(train_df))
test_preds_gm = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train_df[train_df['wheezy-copper-turtle-magic']==i]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
max_predicts_count = len(train2p)
 test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 ms, ps, n_clusters = get_mean_cov(train3p,train2p['target'].values)
 gm = GaussianMixture(n_components=n_clusters * 2, init_params='random', covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3,test3],axis = 0))
predicts = gm.predict_proba(train3)
 preds_gm[idx1] = np.sum(predicts[:,:n_clusters], axis=1)
 predicts = gm.predict_proba(test3)
 test_preds_[idx2] = np.sum(predicts[:,:n_clusters], axis=1)
 auc = roc_auc_score(train_df['target'], preds_gm)
 print('QDA scores CV =',round(auc,5))<save_to_csv>
final_result1 = pd.DataFrame({'Id': df_test['Id'], 'SalePrice': final_labels1} )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = test_preds_
sub.to_csv('submission.csv',index=False )<set_options>
final_result1.to_csv('submission_reg.csv', index=False)
 print("Your submission was successfully saved!" )
warnings.filterwarnings('ignore')
<load_from_csv>
submission_reg = pd.read_csv("submission_reg.csv")
 submission_reg
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
model = Sequential()
model.add(Dense(331,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(165,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse' )
%%time
NFOLDS=11
RS=42
oof=np.zeros(len(train))
preds=np.zeros(len(test))
magic_max=train['wheezy-copper-turtle-magic'].max()
magic_min=train['wheezy-copper-turtle-magic'].min()
auc_all=np.array([])
 print(f"Computing centroids and covariances for the four clusters(two per class ).")
 for i in tqdm(range(magic_min, magic_max+1)) :
X = train[train['wheezy-copper-turtle-magic']==i].copy()
Y = X.pop('target' ).values
X_test = test[test['wheezy-copper-turtle-magic']==i].copy()
idx_train = X.index
idx_test = X_test.index
X.reset_index(drop=True,inplace=True)
 cols = [c for c in X.columns if c not in ['id', 'wheezy-copper-turtle-magic']]
X = X[cols].values
X_test = X_test[cols].values
vt = VarianceThreshold(threshold=1.5 ).fit(X)
 X = vt.transform(X)
 X_test = vt.transform(X_test)
 auc_folds=np.array([])
 folds = StratifiedKFold(n_splits=NFOLDS, random_state=RS)
 for fold_num,(train_index, val_index)in enumerate(folds.split(X, Y), 1):
X_train, Y_train = X[train_index, :], Y[train_index]
X_val, Y_val = X[val_index, :], Y[val_index]
X_train_0 = X_train[Y_train==0]
Y_train_0 = Y_train[Y_train==0].reshape(-1, 1)
 X_train_1 = X_train[Y_train==1]
Y_train_1 = Y_train[Y_train==1].reshape(-1, 1)
 params={'n_components' : 2,
'init_params': 'random',
'covariance_type': 'full',
'tol':0.001,
'reg_covar': 0.001,
'max_iter': 100,
'n_init': 10,
}
clf_0 = GaussianMixture(**params)
 clf_0.fit(X_train_0)
 means_0 = clf_0.means_
covs_0 = clf_0.covariances_
ps_0 = [np.linalg.inv(m)for m in covs_0]
clf_1 = GaussianMixture(**params)
 clf_1.fit(X_train_1)
 means_1 = clf_1.means_
covs_1 = clf_1.covariances_
ps_1 = [np.linalg.inv(m)for m in covs_1]
ms = np.stack(( means_0[0], means_0[1], means_1[0], means_1[1]))
ps = np.stack(( ps_0[0], ps_0[1], ps_1[0], ps_1[1]))
params={'n_components' : 4,
'init_params': 'random',
'covariance_type': 'full',
'tol':0.001,
'reg_covar': 0.001,
'max_iter': 100,
'n_init': 10,
'means_init': ms,
'precisions_init': ps,
}
clf = GaussianMixture(**params)
 clf.fit(np.concatenate([X_train, X_test], axis = 0))
oof[idx_train[val_index]] = np.sum(clf.predict_proba(X_val)[:, 2:], axis=1)
 preds[idx_test] += np.sum(clf.predict_proba(X_test)[:,2: ], axis=1)/NFOLDS
auc = roc_auc_score(Y_val, oof[idx_train[val_index]])
 auc_folds = np.append(auc_folds, auc)
 auc_all = np.append(auc_all, np.mean(auc_folds))
auc_combo = roc_auc_score(train['target'].values, oof)
 auc_folds_average = np.mean(auc_all)
 std = np.std(auc_all)/(np.sqrt(NFOLDS)*np.sqrt(magic_max+1))
print(f'The combined AUC CV score is {round(auc_combo,5)}.')
 print(f'The folds average AUC CV score is {round(auc_folds_average,5)}.')
 print(f'The standard deviation is {round(std, 5)}.' )<save_to_csv>
early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)
 model.fit(x=train_X,y=train_y,
validation_data=(val_X,val_y),
batch_size=128,epochs=400, callbacks=[early_stop] )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<set_options>
losses = pd.DataFrame(model.history.history)
 losses.plot()
warnings.filterwarnings('ignore' )<load_from_csv>
predictions2 = model.predict(val_X)
 predictions2
n_folds = 5
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )<data_type_conversions>
predictions3 = model.predict(X_test)
 predictions3
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps<concatenate>
predictions3=predictions3.flatten()
predictions3
oof = np.zeros(len(train))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=n_folds, random_state=42)
 for train_index, val_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[val_index]] = clf.predict_proba(train3[val_index,:])[:,1]
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}' )<feature_engineering>
final_labels2 = predictions3
final_result2 = pd.DataFrame({'Id': df_test['Id'], 'SalePrice': final_labels2} )
new_train = train.copy()
new_train.loc[oof > 0.98, 'target'] = 1
new_train.loc[oof < 0.02, 'target'] = 0<define_variables>
final_result2.to_csv('submission_keras.csv', index=False)
 print("Your submission was successfully saved!" )
oof_gm = np.zeros(len(train))
oof_bc_qda = np.zeros(len(train))
oof_bc_knn = np.zeros(len(train))
oof_svc = np.zeros(len(train))
preds_gm = np.zeros(len(test))
preds_bc_qda = np.zeros(len(test))
preds_bc_knn = np.zeros(len(test))
preds_svc = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=n_folds, random_state=42)
 for train_index, val_index in skf.split(train2, train2['target']):
clf_qda = QuadraticDiscriminantAnalysis(0.5)
 clf = BaggingClassifier(base_estimator=clf_qda, n_estimators=40, n_jobs=-1, random_state=42)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof_bc_qda[idx1[val_index]] = clf.predict_proba(train3[val_index,:])[:,1]
preds_bc_qda[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf_knn = KNeighborsClassifier(n_neighbors=3, n_jobs=-1)
 clf = BaggingClassifier(base_estimator=clf_knn, n_estimators=40, n_jobs=-1, random_state=42)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof_bc_knn[idx1[val_index]] = clf.predict_proba(train3[val_index,:])[:,1]
preds_bc_knn[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf_svc = SVC(random_state=42, probability=True)
 clf_svc.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof_svc[idx1[val_index]] = clf_svc.predict_proba(train3[val_index,:])[:,1]
preds_svc[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gm = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001, reg_covar=0.001, max_iter=100, n_init=1, means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof_gm[idx1[val_index]] = gm.predict_proba(train3[val_index,:])[:,0]
oof_gm[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train['target'], oof_bc_qda)
 print(f'AUC bc qda: {auc:.5}')
 auc = roc_auc_score(train['target'], oof_svc)
 print(f'AUC svc: {auc:.5}')
 auc = roc_auc_score(train['target'], oof_bc_knn)
 print(f'AUC bc knn: {auc:.5}')
 auc = roc_auc_score(train['target'], oof_gm)
 print(f'AUC gm: {auc:.5}' )<compute_test_metric>
submission_keras = pd.read_csv("submission_keras.csv")
 submission_keras
preds_blend1 = 0.68*preds_gm + 0.32*preds_svc
preds_blend = 0.55*preds_bc_qda + 0.25*preds_blend1 + 0.2*preds_bc_knn
oof_blend1 = 0.68*oof_gm + 0.32*oof_svc
oof_blend = 0.55*oof_bc_qda + 0.25*oof_blend1 + 0.2*oof_bc_knn
auc = roc_auc_score(train['target'], oof_blend)
 print(f'AUC: {auc:.7}' )<feature_engineering>
submission_avg =(submission_lasso + submission_reg + submission_keras)/3
for itr in range(2):
test['target'] = preds_blend
test.loc[test['target'] > 0.60, 'target'] = 1
test.loc[test['target'] < 0.40, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 oof_gm = np.zeros(len(train))
oof_bc_qda = np.zeros(len(train))
oof_bc_knn = np.zeros(len(train))
oof_svc = np.zeros(len(train))
preds_gm = np.zeros(len(test))
preds_bc_qda = np.zeros(len(test))
preds_knn = np.zeros(len(test))
preds_svc = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=n_folds, random_state=42)
 for train_index, val_index in skf.split(train2, train2['target']):
oof_val_index = [t for t in val_index if t < len(idx1)]
clf_qda = QuadraticDiscriminantAnalysis(0.5)
 clf = BaggingClassifier(base_estimator=clf_qda, n_estimators=40, n_jobs=-1, random_state=42)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_val_index)> 0:
oof_bc_qda[idx1[oof_val_index]] = clf.predict_proba(train3[oof_val_index,:])[:,1]
preds_bc_qda[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf_knn = KNeighborsClassifier(n_neighbors=3, n_jobs=-1)
 clf = BaggingClassifier(base_estimator=clf_knn, n_estimators=40, n_jobs=-1, random_state=42)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_val_index)> 0:
oof_bc_knn[idx1[oof_val_index]] = clf.predict_proba(train3[oof_val_index,:])[:,1]
preds_bc_knn[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = SVC(random_state=42, probability=True)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_val_index)> 0:
oof_svc[idx1[oof_val_index]] = clf.predict_proba(train3[oof_val_index,:])[:,1]
preds_svc[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gm = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001, reg_covar=0.001, max_iter=100, n_init=1, means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
if len(oof_val_index)> 0:
oof_gm[idx1[oof_val_index]] = gm.predict_proba(train3[oof_val_index,:])[:,0]
preds_gm[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train['target'], oof_bc_qda)
 print(f'AUC bc qda: {auc:.5}')
 auc = roc_auc_score(train['target'], oof_svc)
 print(f'AUC svc: {auc:.5}')
 auc = roc_auc_score(train['target'], oof_bc_knn)
 print(f'AUC bc knn: {auc:.5}')
 auc = roc_auc_score(train['target'], oof_gm)
 print(f'AUC gm: {auc:.5}')
 preds_blend1 = 0.68*preds_gm + 0.32*preds_svc
preds_blend = 0.55*preds_bc_qda + 0.25*preds_blend1 + 0.2*preds_bc_knn
oof_blend1 = 0.68*oof_gm + 0.32*oof_svc
oof_blend = 0.55*oof_bc_qda + 0.25*oof_blend1 + 0.2*oof_bc_knn
auc = roc_auc_score(train['target'], oof_blend)
 print(f'AUC blend: {auc:.7}' )<save_to_csv>
my_submission = pd.DataFrame()
my_submission['Id'] = df_test.Id
my_submission['SalePrice'] = submission_avg.SalePrice
my_submission.to_csv('my_submission.csv',index=False )
sub = pd.read_csv('.. /input/sample_submission.csv')
 if len(test)!= 131073:
sub['target'] = preds_blend
sub.to_csv('submission.csv',index=False )<set_options>
my_submission = pd.read_csv("my_submission.csv")
 my_submission