Kaggle competition: House Prices - Advanced Regression Techniques
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 sub['FVC1'] = 0.996*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<save_to_csv>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 data = pd.concat([train, test],ignore_index=True, sort=False )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
cor[abs(cor['SalePrice'].values)>= 0.5]['SalePrice'].sort_values(ascending=False)[1:]
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<define_variables>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<12.5)].index)
 train.reset_index(drop = True, inplace = True )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 return seed<load_from_csv>
data = pd.concat(( train, test)).reset_index(drop=True )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
columns = data.columns
for col in columns:
print(data[col].value_counts())
 print('
' )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
missing_tot = data.isnull().sum().sort_values(ascending = False)
 missing_per =(( data.isnull().sum() /data.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([missing_tot, missing_per] , axis=1, keys=['Total', 'Percent'] )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
data_cat.isnull().sum().sort_values(ascending =False )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
d = data_cat.isnull().sum().sort_values(ascending =False)
 d.index
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
columns1 = ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageCond',
'GarageQual', 'GarageFinish', 'GarageType', 'BsmtCond', 'BsmtExposure',
'BsmtQual', 'BsmtFinType2', 'BsmtFinType1', 'MasVnrType']
columns2 = ['MSZoning','Functional', 'Utilities', 'Electrical', 'KitchenQual', 'SaleType',
'Exterior2nd', 'Exterior1st']
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
for col in columns1:
data_cat[col].fillna('None',inplace =True )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
for col in columns2:
data_cat[col].fillna(data[col].mode() [0],inplace = True )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE'] 
<filter>
d2 = data_int.isnull().sum().sort_values(ascending=False)
 d2.index
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<choose_model_class>
columns3 = ['LotFrontage', 'GarageYrBlt', 'MasVnrArea', 'BsmtFullBath',
'BsmtHalfBath', 'TotalBsmtSF', 'BsmtUnfSF', 'GarageArea', 'GarageCars',
'BsmtFinSF2', 'BsmtFinSF1']
SEED = seed_everything(42)
 NFOLD = 5
EPOCHS = 800
BATCH_SIZE = 128
M_LOSS = 0.775
LR = 0.1
DECAY = 0.01
kf = KFold(n_splits=NFOLD )<compute_test_metric>
for col in columns3:
data_int[col].fillna(0,inplace=True )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.775), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
df = pd.concat([data_cat,data_int],axis=1 )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
delta = np.zeros(( z.shape[0], 3))<train_model>
df['MSSubClass'] = df['MSSubClass'].apply(str)
 df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str)
 df.drop('SalePrice', axis=1, inplace = True )
%%time
cnt = 0
train = []
val = []
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs= EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print()
pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
delta += net.predict(z)/ NFOLD 
<compute_test_metric>
missing_tot = df.isnull().sum().sort_values(ascending = False)
 missing_per =(( df.isnull().sum() /df.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([missing_tot, missing_per] , axis=1, keys=['Total', 'Percent'])
 missing_data.head()
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean)
<compute_test_metric>
numeric_feats = df.dtypes[df.dtypes != "object"].index
skewed_feats = df[numeric_feats].apply(lambda x: x.skew() ).sort_values(ascending=False)
 skewed_feats
o_clipped = np.maximum(delta[:,2] - delta[:,0], 70)
 delta = np.minimum(np.abs(delta[:, 1] - y), 1000)
 sqrt =(np.sqrt(( 2)))
 score =(-(sqrt *(delta)) /(o_clipped)) - tf.math.log(sqrt * o_clipped)
 logL_Score = np.mean(score )<compute_train_metric>
high_skew = skewed_feats[skewed_feats > 0.5]
skew_index = high_skew.index
for i in skew_index:
df[i] = boxcox1p(df[i], boxcox_normmax(df[i] + 1))
print('we are using fix seed value always to avoid RANDOMIZATION(NEED TO GET SAME RESULT)')
 print('Seed value =',SEED)
 print('Number of folds =',BATCH_SIZE)
 print('Number of epochs =',EPOCHS)
 print('
mean_absolute_error =',sigma_opt)
 print('unc_mean =',unc.mean())
 print()
print('Log_laplace_Scores =',logL_Score)
 print()
print('unc_min =',unc.min())
 print('unc_max =',unc.max())
 print('unc_mean =',(unc>=0 ).mean() )<create_dataframe>
final_features = pd.get_dummies(df ).reset_index(drop=True)
 print('Features size:', df.shape)
 final_features.head()
data = [[index, logL_Score, sigma_opt, unc.mean() , unc.min() , unc.max() ,(unc>=0 ).mean() ,
BATCH_SIZE, EPOCHS, NFOLD, M_LOSS, LR, DECAY ,SEED]]
columns = ['S.No','Score', 'meanAbseErr', 'unc.mean', 'unc.min', 'unc.max', '(unc>=0)',
'Bsize', 'epoch', 'NFOLD', 'M_LOSS', 'LR', 'DECAY', 'seed']
kernal_stats = pd.DataFrame(data, columns=columns)
 kernal_stats<save_to_csv>
nrow_train = train.shape[0]
X_train = final_features[:nrow_train]
X_test = final_features[nrow_train:]
Y = train['SalePrice']
stats = pd.concat([stats, kernal_stats])
 stats.to_csv('kernal.csv', index = False)
 index+=1
stats 
<prepare_output>
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7,
alphas=alphas2,random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7,
alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
print('Elasticnet')
 elastic_model = elasticnet.fit(X_train, Y)
 print('Lasso')
 lasso_model = lasso.fit(X_train, Y)
 print('Ridge')
 ridge_model = ridge.fit(X_train, Y )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
def blend_models(X):
return(( elastic_model.predict(X)) +(lasso_model.predict(X)) +(ridge_model.predict(X)))/3
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
submission = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.expm1(blend_models(X_test))
<save_to_csv><EOS> 
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission_regression1.csv", index=False )