Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
%matplotlib inline
py.init_notebook_mode(connected = False)
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128<load_from_csv>
traindf = pd.read_csv('.. /input/train.csv')
 testdf = pd.read_csv('.. /input/test.csv')
 print('The size of traind data is:{}'.format(traindf.shape),'
','The size of test data is:{}'.format(testdf.shape))
traindf.head()
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
nulls = traindf.isnull().sum()
nulls[nulls > 1000]
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
traindf.drop(traindf.loc[(traindf['GrLivArea'] > 4500)&(traindf['SalePrice'] < 300000)].index, axis = 0, inplace = True)
 traindf.drop(traindf.loc[(traindf['LotArea'] > 150000)&(traindf['SalePrice'] < 400000)].index, axis = 0, inplace = True )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
print('The above Histogram has skew: {} and kurtosis: {}'.format(skew(traindf['SalePrice']), kurtosis(traindf['SalePrice'])) )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
def fulldf(data1, data2):
fulldf = pd.concat([data1, data2])
 return fulldf
def sepdf(data):
data1 = fulldf.iloc[:len(traindf)]
data2 = fulldf.iloc[len(traindf):]
return data1, data2
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
target = traindf['SalePrice']
Id = testdf['Id']
fulldf = fulldf(traindf.drop('SalePrice', axis = 1), testdf )
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
features = fulldf.select_dtypes(include = ['object'])
 f = pd.DataFrame()
for fe in features.columns:
l = 0
l = []
l.append(fulldf[fe].unique())
 f[fe] = l
features_dict = {}
for column in f.columns:
new_column = 0
new_column = pd.Series(f[column][0], name = column,)
 keys = 0
keys = range(len(new_column))
for i in keys:
features_dict[column] = new_column
unique_feature_data = pd.DataFrame(features_dict)
 unique_feature_data.fillna(' ', inplace = True)
 pd.set_option('max_columns',None)
 print(' Unique Categorical Features')
 unique_feature_data.head()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
fulldf[['PoolQC','PoolArea',
'GarageQual','GarageArea',
'FireplaceQu','Fireplaces',
'BsmtQual','TotalBsmtSF']].sort_values(by = ['PoolQC','GarageQual','FireplaceQu','BsmtQual','TotalBsmtSF'], na_position = 'first' ).head()
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
missing = fulldf.isnull().sum()
missing = missing[missing > 0]
Smallmissing = missing[missing < 20]
for column in Smallmissing.index:
fulldf[column].fillna(fulldf[column].mode() [0], inplace = True )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
fulldf[['GarageYrBlt', 'GarageType', 'GarageCond', 'GarageFinish', 'GarageArea']].sort_values(by = ['GarageYrBlt', 'GarageType',
'GarageCond', 'GarageFinish'], na_position = 'first' ).head(3 )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.775), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
Bsmt = ['BsmtFinType1', 'BsmtFinType2', 'BsmtCond', 'BsmtExposure',]
for B in Bsmt:
fulldf[B].fillna('None', inplace = True )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))<choose_model_class>
k = ['LotFrontage','MasVnrType', 'MasVnrArea', 'Fence', 'MiscFeature']
for K in k:
fulldf.loc[:,K].fillna(fulldf[K].mode() [0], inplace = True )
NFOLD =5
kf = GroupKFold(n_splits=NFOLD)
 split_groups = tr['Patient']<train_model>
fulldf.loc[:,['Alley', 'Street']].sort_values(by = 'Alley', na_position = 'first' ).head(3 )
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z,groups=split_groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD 
<compute_test_metric>
fulldf.drop(['Id', 'Alley'], axis = 1, inplace = True )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<compute_test_metric>
ordinal_fea = ['BsmtCond', 'BsmtQual', 'ExterCond', 'ExterQual', 'FireplaceQu', 'GarageCond', 'GarageQual', 'HeatingQC','KitchenQual', 'PoolQC']
for i in ordinal_fea:
fulldf[i] = fulldf[i].replace(['Ex','Fa','Gd','TA','Po','None'],[5, 4, 3, 2, 1, 0] )
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 print(np.mean(score))<prepare_output>
num_fea = fulldf.select_dtypes(exclude = ['object'] ).columns
skewed_fea = fulldf[num_fea].apply(lambda x: skew(x))
skewed_col_names = skewed_fea.index
skewed_col_names = skewed_col_names.str.contains('(Area)|(SF)|(Frontage)')
 for col_name in skewed_fea[skewed_col_names].index:
fulldf[col_name] = np.log1p(fulldf[col_name] )
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
fulldf['HouseAge'] = fulldf['YearRemodAdd'] - fulldf['YearBuilt']
fulldf['CompleteHouseArea'] = fulldf['LotArea'] + fulldf['MasVnrArea'] + fulldf['GrLivArea'] + fulldf['GarageArea'] + fulldf['PoolArea'] + fulldf['TotalBsmtSF']
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
num_cat_fea = list(((( set(num_fea)) - set(ordinal_fea)) - set(skewed_fea[skewed_col_names].index)))
 Le = LabelEncoder()
for col_name in num_cat_fea:
fulldf[col_name] = Le.fit_transform(fulldf[col_name] )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
cat_fea = list(set(unique_feature_data.columns[1:])- set(num_fea))
print('The shapes of train and test Dataframes before conversion is respectively:({}),({})'.format(traindf.shape, testdf.shape))
fulldf = pd.get_dummies(fulldf, columns = cat_fea, drop_first = True)
 traindf, testdf = sepdf(fulldf)
 print('The shapes of train and test Dataframes After conversion is respectively:({}),({})'.format(traindf.shape, testdf.shape))
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.linear_model import Lasso, Ridge, ElasticNet
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.pipeline import make_pipeline
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_Quantile_Regression.csv", index=False )<load_from_csv>
scorer = make_scorer(mean_squared_error, greater_is_better = False)
 def RMSE_CV(model):
RMSE = np.sqrt(-cross_val_score(model, traindf, target, scoring = scorer, cv = 5))
return RMSE
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 submission = pd.read_csv(f"{ROOT}/sample_submission.csv")
 submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission = submission[['Patient','Weeks','Confidence','Patient_Week']]
submission = submission.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
alphas = [10, 15, 20, 30, 32, 35, 50, 75]
ridge_cv = []
for alpha in alphas:
ridge = make_pipeline(StandardScaler() , RobustScaler() , Ridge(alpha = alpha))
ridge_cv.append(RMSE_CV(ridge ).mean())
 ridge_cv = pd.Series(ridge_cv, index = alphas)
 print('The minimum error is {:.4f} at {:.4f}'.format(ridge_cv.min() , ridge_cv.idxmin()))
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
submission['WHERE'] = 'test'
data = tr.append([chunk, submission] )<feature_engineering>
alphas = [0.1, 0.15, 0.2, 0.005, 0.0032, 0.0030, 0.0024,]
lasso_cv = []
for alpha in alphas:
lasso = make_pipeline(StandardScaler() , RobustScaler() , Lasso(alpha = alpha))
lasso_cv.append(RMSE_CV(lasso ).mean())
 lasso_cv = pd.Series(lasso_cv, index = alphas)
 print('The minimum error is {:.4f} at {:.4f}'.format(lasso_cv.min() , lasso_cv.idxmin()))
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
<merge>
alphas = [0.1, 0.15, 0.2, 0.005, 0.0032, 0.0030, 0.0024, 0.0028]
ENet_cv = []
for alpha in alphas:
ENet = make_pipeline(StandardScaler() , RobustScaler() , ElasticNet(alpha = alpha, l1_ratio = 0.9))
ENet_cv.append(RMSE_CV(ENet ).mean())
 ENet_cv = pd.Series(lasso_cv, index = alphas)
 print('The minimum error is {:.4f} at {:.4f}'.format(ENet_cv.min() , ENet_cv.idxmin()))
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base 
<data_type_conversions>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
averaged_models = AveragingModels(models =(ENet, ridge, lasso))
score = RMSE_CV(averaged_models)
 print('Averaged base models score: {:.4f}'.format(score.mean()))
<filter><EOS> 
averaged_models.fit(traindf, target)
 pred = averaged_models.predict(testdf)
 pred = np.expm1(pred)
 sub = pd.DataFrame({'Id': Id, 'SalePrice': pred})
 sub.to_csv('submission.csv',index = False )