Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
import numpy as np
import pandas as pd
import scipy.special
import matplotlib.pyplot as plt
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_log_error
from sklearn.ensemble import IsolationForest
from xgboost import XGBRegressor
warnings.simplefilter('ignore')
 init_notebook_mode()
sns.set()
%matplotlib inline<load_from_csv>
train_data = pd.read_csv(path_in+'train.csv', index_col=0)
 test_data = pd.read_csv(path_in+'test.csv', index_col=0)
 samp_subm = pd.read_csv(path_in+'sample_submission.csv', index_col=0 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<create_dataframe>
print('number train samples: ', len(train_data.index))
print('number test samples: ', len(test_data.index))
%%time
oof = np.zeros(len(train))
preds = np.zeros(len(test))
params = [{'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]}]
reg_params = np.zeros(512)
 for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
qda = QuadraticDiscriminantAnalysis()
clf = GridSearchCV(qda, params, cv=4)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 reg_params[i] = clf.best_params_['reg_param']
oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits 
<compute_test_metric>
y_train = train_data['SalePrice'].copy().to_frame()
del train_data['SalePrice']
y_train.head()
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
<feature_engineering>
data = pd.concat([train_data, test_data] )
for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
<feature_engineering>
no_features = ['Alley', 'PoolQC', 'Fence', 'MiscFeature', 'FireplaceQu']
data = data[data.columns.difference(no_features)]
t = test.copy()
t['target'] = preds
t.loc[t['target'] > 0.955, 'target'] = 1
t.loc[t['target'] < 0.045, 'target'] = 0
usefull_test = t[(t['target'] == 1)|(t['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0 
<save_to_csv>
cols_with_missing_values = [col for col in data.columns if data[col].isnull().any() ]
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv', index=False )<set_options>
data.dtypes.value_counts()
warnings.filterwarnings('ignore' )<init_hyperparams>
data[cols_with_missing_values].dtypes.value_counts()
n_models = 4
n_splits = 12
seed_split = 123
seed_gm = 987
cov_type = 'GL_0.2'
n_init = 4
init_params = 'random'
n_clusters_per_class = 3<load_from_csv>
for col in cols_with_missing_values:
data[col + '_was_missing'] = data[col].isnull()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train = train.reset_index(drop=True)
 print('train:', train.shape)
 print('test:', test.shape )<compute_train_metric>
for col in cols_with_missing_values:
data[col] = data[col].fillna(data[col].value_counts().index[0] )
def get_cov_estimator(cov_type):
if cov_type == 'LW':
model = LedoitWolf()
elif cov_type == 'OAS':
model = OAS()
elif cov_type == 'MCD':
model = MinCovDet()
elif cov_type[:2] == 'SC':
shrinkage = float(cov_type.split('_')[1])
 model = ShrunkCovariance(shrinkage=shrinkage)
 elif cov_type[:2] == 'GL':
alpha = float(cov_type.split('_')[1])
 model = GraphicalLasso(alpha=alpha)
 return model
def get_mean_cov(x, y, n_clusters_per_class=1, cov_type='LW'):
model = get_cov_estimator(cov_type)
 ones =(y == 1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y == 0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1] * n_clusters_per_class + [m2] * n_clusters_per_class)
 ps = np.stack([p1] * n_clusters_per_class + [p2] * n_clusters_per_class)
 return ms, ps<define_variables>
cols_for_one_hot_encdoing = []
cols_for_scale = []
for col in data.columns:
if data[col].dtypes == 'object':
cols_for_one_hot_encdoing.append(col)
 elif data[col].dtypes != 'bool':
cols_for_scale.append(col )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof_avg = np.zeros(len(train))
preds = np.zeros(len(test))
for j in range(n_models):
t1 = time.time()
oof = np.zeros(len(train))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=n_splits,
random_state=seed_split + j,
shuffle=True)
 for train_idx, valid_idx in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_idx, :],
train2.loc[train_idx]['target'].values,
n_clusters_per_class=n_clusters_per_class,
cov_type=cov_type)
 gm = GaussianMixture(n_components=2 * n_clusters_per_class,
init_params=init_params,
covariance_type='full',
max_iter=100,
n_init=n_init,
precisions_init=ps,
random_state=seed_gm + i)
 gm.fit(np.concatenate([train3[train_idx, :], test3]))
oof[idx1[valid_idx]] += gm.predict_proba(train3[valid_idx, :])[:, :2].sum(1)
 preds[idx2] += gm.predict_proba(test3)[:, :2].sum(1)
 oof_avg += oof
auc_one = roc_auc_score(train['target'], oof)
 auc_avg = roc_auc_score(train['target'], oof_avg)
 print(f' model:{j} AUC_avg:{auc_avg:.5f} AUC:{auc_one:.5f} time:{time.time() - t1:.1f}s' )<save_to_csv>
data = pd.get_dummies(data, columns = cols_for_one_hot_encdoing, prefix = cols_for_one_hot_encdoing )
preds /= n_models * n_splits
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv', index=False )<import_modules>
import torch
import torch.nn as nn
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import transformers
from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup<set_options>
X_train = data.loc[train_data.index]
X_test = data.loc[test_data.index]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<set_options>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=2020 )
SEED = 1234
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True<load_from_csv>
model_XGB = XGBRegressor(objective='reg:squarederror', n_estimators=1000)
 model_XGB.fit(X_train, y_train)
 y_val_pred = model_XGB.predict(X_val)
 np.sqrt(mean_squared_log_error(y_val, y_val_pred))
train_df = pd.read_csv("/kaggle/input/quora-question-pairs/train.csv.zip")
 train_df.info()<correct_missing_values>
y_test = model_XGB.predict(X_test ).astype(int )
<string_transform><EOS> 
output = pd.DataFrame({'Id': samp_subm.index,
'SalePrice': y_test})
 output.to_csv('submission.csv', index=False )