Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<compute_test_metric>
%matplotlib inline
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
warnings.filterwarnings("ignore")
 assert 'house-prices-advanced-regression-techniques' in os.listdir('.. /input/'), 'House Prices dataset not loaded!'
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="elu", name="d2" )(z)
 x = L.Dense(100, activation="relu", name="d3" )(z)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
HOME_FOLDER = '.. /input/house-prices-advanced-regression-techniques/'
df_train = pd.read_csv(HOME_FOLDER + 'train.csv', index_col='Id')
 df_test = pd.read_csv(HOME_FOLDER + 'test.csv', index_col='Id' )
y = tr['FVC'].values
z = tr[FE].values
ze = submission[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))<choose_model_class>
n_train, n_test = df_train.shape[0], df_test.shape[0]
df_full = pd.concat(( df_train, df_test), sort=False ).reset_index(drop=True)
 y_train = df_train['SalePrice']
df_full.drop(['SalePrice'], axis=1, inplace=True )
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient'] 
<train_model>
missing_ratio =(df_full.isnull().sum() / df_full.shape[0] * 100 ).sort_values(ascending=False)
 missing_ratio.drop(missing_ratio[missing_ratio == 0].index, inplace=True)
 missing_values = pd.DataFrame(missing_ratio, columns=['Missing Ratio'])
 print(missing_values )
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD 
<compute_test_metric>
df_full['PoolQC'].fillna('None', inplace=True )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<compute_test_metric>
df_full['MiscFeature'].fillna('None', inplace=True )
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))<feature_engineering>
cols = ['Alley', 'Fence', 'FireplaceQu']
df_full[cols] = df_full[cols].fillna('None', inplace=False )
submission['FVC1'] = pe[:, 1]
submission['Confidence1'] = pe[:, 2] - pe[:, 0]<create_dataframe>
df_full['LotFrontage'] = df_full.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
submission1 = submission[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
cols = ['GarageCond', 'GarageQual', 'GarageFinish', 'GarageType']
df_full[cols] = df_full[cols].fillna('None' )
submission1.loc[~submission1.FVC1.isnull() ,'FVC'] = submission1.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
submission1['Confidence'] = sigma_opt
else:
submission1.loc[~submission1.FVC1.isnull() ,'Confidence'] = submission1.loc[~submission1.FVC1.isnull() ,'Confidence1']<load_from_csv>
cols = ['GarageYrBlt', 'GarageArea', 'GarageCars']
df_full[cols] = df_full[cols].fillna(0 )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
submission1.loc[submission1['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission1.loc[submission1['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
cols = ['BsmtCond', 'BsmtExposure', 'BsmtQual', 'BsmtFinType2', 'BsmtFinType1']
df_full[cols] = df_full[cols].fillna('None' )
submission1[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression_V2.csv", index=False )<load_from_csv>
cols = ['BsmtHalfBath', 'BsmtFullBath', 'BsmtFinSF2', 'BsmtFinSF1', 'BsmtUnfSF', 'TotalBsmtSF']
df_full[cols] = df_full[cols].fillna(0 )
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 submission2 = pd.read_csv(f"{ROOT}/sample_submission.csv")
 submission2['Patient'] = submission2['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission2['Weeks'] = submission2['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission2 = submission2[['Patient','Weeks','Confidence','Patient_Week']]
submission2 = submission2.merge(test.drop('Weeks', axis=1), on="Patient" )<concatenate>
print(df_full[['MasVnrType', 'MasVnrArea']].loc[df_full['MasVnrType'].isnull() & df_full['MasVnrArea'].notnull() ])
 print('
Houses with price between 196.000 and 200.000:')
 print(df_full[['MasVnrType']].loc[(df_full['MasVnrArea'] <= 200.000)&(df_full['MasVnrArea'] >= 196.000)].squeeze().value_counts() )
train['WHERE'] = 'train'
test['WHERE'] = 'val'
submission2['WHERE'] = 'test'
data = train.append([test, submission2] )<feature_engineering>
mask =(df_full['MasVnrType'].isnull())&(df_full['MasVnrArea'].notnull())
 df_full.loc[mask, 'MasVnrType'] = df_full.loc[mask, 'MasVnrType'].fillna('None')
 df_full['MasVnrType'].fillna('None', inplace=True)
 df_full['MasVnrArea'].fillna(0, inplace=True )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
df_full['MSZoning'].fillna(df_full['MSZoning'].mode() [0], inplace=True )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
df_full.drop(['Utilities'], axis=1, inplace=True )
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
df_full['Functional'].fillna('Typ', inplace=True )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
assert df_full.isnull().sum().sort_values(ascending=False)[0] == 0, 'Mmm, there are still missing values! Check again.'
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
df_full['MSSubClass'] = df_full['MSSubClass'].apply(str)
 df_full['OverallCond'] = df_full['OverallCond'].apply(str)
 def mapper(month):
date = datetime.datetime(2000, month, 1)
 return date.strftime('%b')
 df_full['MoSold'] = df_full['MoSold'].apply(mapper )
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
submission2 = data.loc[data.WHERE=='test']
del data<compute_test_metric>
cols = ['MSSubClass', 'Street', 'Alley','LandSlope', 'OverallQual', 'OverallCond',
'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'HeatingQC', 'CentralAir', 'KitchenQual', 'Functional',
'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive',
'PoolQC', 'Fence', 'YrSold', 'MoSold']
for c in cols:
le = LabelEncoder()
le.fit(list(df_full[c].values))
df_full[c] = le.transform(list(df_full[c].values))
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.11, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model 
<prepare_x_and_y>
skewness = skewness[abs(skewness['Skew'])> 0.75]
print(f'There are {skewness.shape[0]} skewed numerical features to Box Cox transform.')
 lambda_param = 0.15
for feature in skewness.index:
df_full[feature] = boxcox1p(df_full[feature], lambda_param )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))<choose_model_class>
df_full = pd.get_dummies(df_full)
 print(f'Now there are {df_full.shape[1]} features.' )
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']<train_model>
X_train = df_full[:n_train]
X_test = df_full[n_train:]
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD 
<compute_test_metric>
from sklearn.model_selection import KFold, cross_val_score
from xgboost import XGBRegressor
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))<feature_engineering>
def model_validation(model, X, y, n_folds=5):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X.values)
 score = -cross_val_score(model, X.values, y, scoring='neg_mean_squared_error', cv=kf)
 return np.sqrt(score )
submission2['FVC1'] = pe[:, 1]
submission2['Confidence1'] = pe[:, 2] - pe[:, 0]<create_dataframe>
model_xgb = XGBRegressor(colsample_bytree=0.8, subsample=0.5,
learning_rate=0.05, max_depth=3,
min_child_weight=1.8, n_estimators=2000,
reg_alpha=0.1, reg_lambda=0.8, gamma=0.01,
silent=1, random_state=7, n_jobs=-1,
early_stopping_rounds=10)
 score = model_validation(model_xgb, X_train, y_train)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
submission2 = submission2[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
model_xgb.fit(X_train, y_train)
 predictions = np.expm1(model_xgb.predict(X_test))
<load_from_csv><EOS> 
output = pd.DataFrame({'Id': df_test.index,
'SalePrice': predictions})
 output.to_csv('submission.csv', index=False )