Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<filter>
warnings.filterwarnings('ignore' )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']<compute_test_metric>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head()
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss<init_hyperparams>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 test.head()
LR = 0.1
BETA_1 = 0.9
BETA_2 = 0.999
DECAY = 0.01
AMSGRAD = False<choose_model_class>
cor_target = abs(train.corr() ["SalePrice"] ).sort_values(ascending=False)
 cor_target
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="NN-Optimized")
 model.compile(loss=mloss(0.8), optimizer = tf.keras.optimizers.Adam(lr = LR, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = None, decay = DECAY, amsgrad = AMSGRAD), metrics=[score])
 return model<prepare_x_and_y>
train.sort_values(by='TotalBsmtSF', ascending=False ).head()
train.sort_values(by='GrLivArea', ascending=False ).head()
y = tr['FVC'].astype('float32' ).values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
train = train.drop(train[train['Id'] == 524].index)
 train = train.drop(train[train['Id'] == 1299].index)
 train.shape, test.shape
NFOLD = 10
kf = KFold(n_splits=NFOLD )<train_model>
y = train['SalePrice'].reset_index(drop=True)
 all_data = pd.concat([train, test] ).reset_index(drop=True )
%%time
count = 0
for train_idx, val_idx in kf.split(z):
count += 1
print(f"FOLD {count}:")
 net = make_model()
net.fit(z[train_idx], y[train_idx], batch_size=BATCH_SIZE, epochs=850,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("Train:", net.evaluate(z[train_idx], y[train_idx], verbose=0, batch_size=BATCH_SIZE))
print("Val:", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("Predicting Test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
all_data.drop(['Id','SalePrice'], axis=1, inplace=True)
 all_data.shape
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss<compute_test_metric>
missingPercentage =(all_data.isnull().sum() * 100 / all_data.shape[0] ).sort_values(ascending=False)
 missingPercentage = missingPercentage[missingPercentage > 0]
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
all_data.drop(columns=["PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu"], inplace=True )
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
columns_zero = ['BsmtFullBath', 'BsmtHalfBath', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'GarageYrBlt',
'GarageArea', 'GarageCars', 'MasVnrArea']
all_data[columns_zero] = all_data[columns_zero].fillna(0 )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<feature_engineering>
columns_none = ['BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','BsmtQual','GarageCond','GarageFinish',
'GarageType','GarageQual']
all_data[columns_none] = all_data[columns_none].fillna("None" )
otest = pd.read_csv(f"{ROOT}/test.csv")
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
all_data['Functional'] = all_data['Functional'].fillna("Typ" )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<set_options>
columns_mode = ['MSZoning', 'MasVnrType', 'Electrical', 'Utilities', 'Exterior1st', 'Exterior2nd', 'SaleType', 'KitchenQual']
for i in columns_mode:
all_data[i] = all_data[i].fillna(all_data[i].mode() [0] )
warnings.filterwarnings("ignore")
<categorify>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data.drop(columns=['MoSold'], inplace=True )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss 
<define_variables>
print('Total missing data points: {}'.format(all_data.isna().sum().sum()))
ROOT = ".. /input/osic-pulmonary-fibrosis-progression" 
<load_from_csv>
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skewed_feats = skewed_feats[abs(skewed_feats)> 0.5].index
high_skewed_feats
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )<concatenate>
for feature in high_skewed_feats:
all_data[feature] = np.log1p(all_data[feature] )
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub] )<feature_engineering>
all_data = pd.get_dummies(all_data ).reset_index(drop=True)
 all_data.shape
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import cross_val_score, KFold
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
kf = KFold(n_splits=10, shuffle=True, random_state=123 )
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<categorify>
lambda_ridge = [5.0, 5.1, 5.2, 5.3, 5.4, 5.5]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas = lambda_ridge, cv= kf))
lb =LabelEncoder()
data["Sex"] = lb.fit_transform(data["Sex"] )<feature_engineering>
lambda_lasso = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
lasso = make_pipeline(RobustScaler() ,
LassoCV(alphas = lambda_lasso, cv= kf, max_iter=1e7))
FE = []
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['Sex','age','percent','week','BASE']<train_on_grid>
lambda_elastic = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
elastic = make_pipeline(RobustScaler() ,
ElasticNetCV(alphas = lambda_elastic, cv= kf, max_iter=1e7))
range_n_clusters = [2,3,4,5,6,7,8,9]
for num_clusters in range_n_clusters:
kmeans = KMeans(n_clusters=num_clusters, max_iter=90)
 kmeans.fit(features_scaled)
 cluster_labels = kmeans.labels_
silhouette_avg = silhouette_score(features_scaled, cluster_labels)
 print("For n_clusters={0}, the silhouette score is {1}".format(num_clusters, silhouette_avg)) 
<train_model>
randfor = RandomForestRegressor(
 n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
oob_score=True,
random_state=123 )
kmeans = KMeans(n_clusters=2, max_iter=3000)
 kmeans.fit(features_scaled)
 features_scaled["Cluster_Id"] =kmeans.labels_
features_scaled["FVC"] =data["FVC"]<train_model>
svmr = make_pipeline(RobustScaler() ,
SVR(
 C=21,
epsilon=0.01,
gamma=0.0003))
algorithm =(KMeans(n_clusters =2,init='k-means++', n_init = 10 ,max_iter=10000,
tol=0.0001, random_state= 111 , algorithm='elkan'))
algorithm.fit(features_scaled)
 data['Cluster_Id'] = algorithm.labels_
del features_scaled<data_type_conversions>
xgb = XGBRegressor(
 learning_rate=0.01,
n_estimators=5000,
max_depth=4,
min_child_weight=0,
subsample=0.7,
colsample_bytree=0.7,
nthread=-1,
seed=123 )
COLS = ['SmokingStatus']
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<categorify>
lgbm = LGBMRegressor(
 n_estimators=5000,
learning_rate=0.01,
num_leaves=6,
max_bin=150,
colsample_bytree=0.3,
min_child_samples=8,
random_state=123,
verbose=-1 )
data = pd.get_dummies(data,columns=["Cluster_Id"])
 for col in("Cluster_Id_0","Cluster_Id_1"):
data[col] = data[col].astype(int)
 FE+=["Cluster_Id_0","Cluster_Id_1"]<init_hyperparams>
score_ridge = -cross_val_score(ridge, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Ridge Regression RMSE score: {}".format(score_ridge))
score_lasso = -cross_val_score(lasso, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Lasso Regression RMSE score: {}".format(score_lasso))
score_elastic = -cross_val_score(elastic, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Elasticnet Regression RMSE score: {}".format(score_elastic))
score_randfor = -cross_val_score(randfor, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Random Forest Regressor RMSE score: {}".format(score_randfor))
score_svmr = -cross_val_score(svmr, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Support Vector Regressor RMSE score: {}".format(score_svmr))
score_xgb = -cross_val_score(xgb, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("XGBoost Regressor RMSE score: {}".format(score_xgb))
score_lgbm = -cross_val_score(lgbm, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Light GBM RMSE score: {}".format(score_lgbm))
features_scaled =data[["min_FVC","percent","BASE","age"]]
inertia = []
algorithm =(DBSCAN(eps=0.6, metric='euclidean',
metric_params=None, algorithm="auto", leaf_size=20, p=None, n_jobs=None))
algorithm.fit(features_scaled)
 labels = algorithm.labels_
core_samples_mask = np.zeros_like(labels, dtype=bool)
 core_samples_mask[algorithm.core_sample_indices_] = True
n_clusters_ = len(set(labels)) -(1 if -1 in labels else 0)
 n_noise_ = list(labels ).count(-1)
 print("Estimated number of clusters: %d" % n_clusters_)
 print("Estimated number of noise points: %d" % n_noise_)
 s = metrics.silhouette_score(features_scaled, algorithm.labels_)
 print(f"Silhouette Coefficient for the Dataset Clusters: {s:.2f}" )<train_model>
scores_df = pd.DataFrame(data=np.array([score_ridge, score_lasso, score_elastic, score_randfor,
score_svmr, score_xgb, score_lgbm]),
index=['Ridge','Lasso','ElasticNet', 'Random Forest', 'Support Vector', 'XGBoost', 'LGBM'],
columns=['Root_Mean_Sq_Err'])
 scores_df = scores_df.sort_values(by='Root_Mean_Sq_Err')
 scores_df
algorithm =(DBSCAN(eps=0.5, min_samples=1, metric='euclidean',
metric_params=None, algorithm="auto", leaf_size=20, p=None, n_jobs=None))
algorithm.fit(features_scaled)
 labels = algorithm.labels_<data_type_conversions>
stacked_model = StackingCVRegressor(
 regressors=(elastic, lasso, ridge, svmr, lgbm, xgb),
meta_regressor=elastic,
use_features_in_secondary=True,
n_jobs=-1 )
data['DBSCAN_CLUSTER'] = algorithm.labels_
for mod in data["DBSCAN_CLUSTER"].unique() :
FE.append(mod)
 data[mod] =(data["DBSCAN_CLUSTER"] == mod ).astype(int)
<filter>
stacked_fit = stacked_model.fit(np.array(X), np.array(y))
y_preds = stacked_fit.predict(np.array(X))
stacked_rmse = mse(y, y_preds, squared=False)
 print('RMSE score of Stacked Model(training data): {}'.format(stacked_rmse))
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<prepare_x_and_y>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission['SalePrice'] = np.floor(np.expm1(stacked_fit.predict(np.array(X_test))))
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1 )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def make_model(learning_rate):
z = L.Input(( 178,), name="Patient")
 x = L.Dense(200, activation="relu",use_bias=True,kernel_regularizer=regularizers.l2(1e-7),
bias_regularizer=regularizers.l2(1e-7),
activity_regularizer=regularizers.l1(1e-7))(z)
 keras.layers.Dropout(0.1)
 x = L.Dense(100, activation="relu",use_bias=True,kernel_regularizer=regularizers.l2(1e-7),
bias_regularizer=regularizers.l2(1e-7),
activity_regularizer=regularizers.l1(1e-7))(x)
 keras.layers.Dropout(0.1)
 x = L.Dense(10, activation="relu",use_bias=True )(x)
 keras.layers.Dropout(0.1)
 p1 = L.Dense(3, activation="linear", name="p1",use_bias=True )(x)
 p2 = L.Dense(3, activation="relu", name="p2",use_bias=True )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model

<choose_model_class>
submission.to_csv("submission.csv", index=False )
<train_model><EOS> 
submission.to_csv("submission.csv", index=False )