Kaggle competition: House Prices - Advanced Regression Techniques
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn import linear_model
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math
import seaborn as sns
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
from math import sqrt
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import RepeatedKFold
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.svm import SVR
from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import LassoCV
from sklearn.linear_model import RidgeCV
from sklearn.linear_model import ElasticNetCV
from sklearn.ensemble import VotingRegressor
from scipy.special import boxcox1p
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
from catboost import CatBoostRegressor
from sklearn.pipeline import make_pipeline
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=265<load_from_csv>
train_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 1000)
 train = train_data.copy()
test = test_data.copy()
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
test = test.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1)
 test.info()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
test_obj.isna().sum()
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
test['SaleType'].value_counts()
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
dataset = [train,test]
columns = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond','Electrical','MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','KitchenQual','Functional','SaleType']
for set in dataset:
for col in columns:
set[col].fillna(set[col].mode() [0],inplace=True)
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
test_num.isna().sum()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
col1 = ['LotFrontage','GarageYrBlt','MasVnrArea','BsmtUnfSF','BsmtFinSF2','BsmtFinSF1','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageCars','GarageArea']
for set in dataset:
for c in col1:
set[c] = set[c].fillna(set[c].median() )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
train = train.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1)
 test = test.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1 )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
dataset = [train,test]
for set in dataset:
set.loc[set['YrSold']<set['YearBuilt'],'YrSold'] = 2009
set['House_Age'] = set['YrSold'] - set['YearBuilt']
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(500, activation="elu", name="d1" )(z)
 x = L.Dense(500, activation="elu", name="d2" )(x)
 p1 = L.Dense(3, activation="elu", name="p1" )(x)
 p2 = L.Dense(3, activation="elu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer="adam", metrics=[score])
 return model<prepare_x_and_y>
train.iloc[1295:1301,:]
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<train_model>
data1 = [train,test]
for set in data1:
set['YrBltAndRemod']=set['YearBuilt']+set['YearRemodAdd']
set['TotalSF']=set['TotalBsmtSF'] + set['1stFlrSF'] + set['2ndFlrSF']
set['Total_sqr_footage'] =(set['BsmtFinSF1'] + set['BsmtFinSF2'] +
set['1stFlrSF'] + set['2ndFlrSF'])
 set['Total_Bathrooms'] =(set['FullBath'] +(0.5 * set['HalfBath'])+
set['BsmtFullBath'] +(0.5 * set['BsmtHalfBath']))
set['Total_porch_sf'] =(set['OpenPorchSF'] + set['3SsnPorch'] +
set['EnclosedPorch'] + set['ScreenPorch'] +
set['WoodDeckSF'])
 set['haspool'] = set['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 set['has2ndfloor'] = set['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 set['hasgarage'] = set['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 set['hasbsmt'] = set['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 set['hasfireplace'] = set['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 col1 = ['MSZoning','BldgType','Exterior2nd']
NFOLD = 3
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<choose_model_class>
train1 = train.copy()
test1 = test.copy()
dataset1 = [train1,test1]
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
Paved = {"N" : 0, "P" : 1, "Y" : 2,'1.5Unf':0,'SFoyer':1,'1.5Fin':2,'2.5Unf':3,'SLvl':4,'1Story':5,'2Story':6,'2.5Fin':7}
MSZ = {'C':0,'RM':1,'RH':2,'RL':3,'FV':4}
LC = {'Bnk':0,'Lvl':1,'Low':2,'HLS':3}
LC1 = {'Inside':0,'FR2':1,'Corner':2,'CulDSac':3,'FR3':4}
LS = {'Gtl':0,'Mod':1,'Sev':2}
N = {'MeadowV':0,'IDOTRR':1,'BrDale':2,'BrkSide':3,'OldTown':4,'Edwards':5,'Blueste':6,'Sawyer':6,'SWISU':7,'NPkVill':7,'NAmes':7,'Mitchel':8,'SawyerW':9,'NWAmes':9,'Gilbert':9,'Blmngtn':9,'ClearCr':10,'CollgCr':10,'Crawfor':11,'Somerst':12,'Veenker':13,'Timber':13,'NridgHt':14,'StoneBr':14,'NoRidge':15}
C1 = {'Artery':0,'RRAe':1,'Feedr':2,'RRAn':3,'RRNe':4,'RRNn':5,'PosN':6,'PosA':7,'Norm':4}
BT = {'2FmCon':0,'Duplx':1,'TwnhsI':2,'TwnhsE':3,'1Fam':4}
MSV = {'BrkCmn':0,'None':1,'BrkFace':2,'Stone':3}
F={'Slab':0,'BrkTil':1,'CBlock':2,'Stone':3,'Wood':4,'PConc':5}
EX2 = {'CBlock':0,'AsbShng':1,'BrkComm':2,'AsphShn':3,'Stone':4,'Stucco':4,'Wd Sdng':4,'MetalSd':4,
'WdShing':5,'Plywood':6,'HdBoard':6,'BrkFace':7,'VinylSd':8,'CemntBd':9,'ImStucc':10,'Other':11}
rest = {'Gambrel':0,'Gable':1,'Mansard':2,'Flat':3,'Hip':4,'Shed':5,'BrkComm':0,'AsphShn':1,
'CBlock':2,'AsbShng':3,'WdShing':4,'Wd Sdng':5,'MetalSd':5,'HdBoard':6,'Stucco':6,'Plywood':7,
'BrkFace':8,'VinylSd':9,'CemntBd':10,'Stone':11,'ImStucc':12,'Mix':0,'FuseP':1,'FuseF':2,
'FuseA':3,'SBrkr':4,'Maj2':0,'Sev':1,'Min1':2,'Min2':2,'Maj1':3,'Mod':4,'Typ':5,'CarPort':0,
'Detchd':1,'2Types':2,'Attchd':4,'BuiltIn':5,'Basment':3,'Unf':0,'RFn':1,'Fin':2,'Oth':0,
'ConLD':1,'COD':2,'ConLw':3,'WD':4,'ConLI':5,'CWD':6,'New':7,'Con':8,'AdjLand':0,'Abnorml':1,
'Family':2,'Normal':3,'Alloca':4,'Partial':5}
for set in dataset1:
set['ExterQual'] = set['ExterQual'].map(bin_map)
 set['ExterCond'] = set['ExterCond'].map(bin_map)
 set['BsmtCond'] = set['BsmtCond'].map(bin_map)
 set['BsmtQual'] = set['BsmtQual'].map(bin_map)
 set['HeatingQC'] = set['HeatingQC'].map(bin_map)
 set['KitchenQual'] = set['KitchenQual'].map(bin_map)
 set['FireplaceQu'] = set['FireplaceQu'].map(bin_map)
 set['GarageQual'] = set['GarageQual'].map(bin_map)
 set['GarageCond'] = set['GarageCond'].map(bin_map)
 set['CentralAir'] = set['CentralAir'].map(bin_map)
 set['LotShape'] = set['LotShape'].map(bin_map)
 set['BsmtExposure'] = set['BsmtExposure'].map(bin_map)
 set['BsmtFinType1'] = set['BsmtFinType1'].map(bin_map)
 set['BsmtFinType2'] = set['BsmtFinType2'].map(bin_map)
 set['PavedDrive'] = set['PavedDrive'].map(Paved)
 set['MSZoning'] = set['MSZoning'].map(MSZ)
 set['LandContour'] = set['LandContour'].map(LC)
 set['LotConfig'] = set['LotConfig'].map(LC1)
 set['LandSlope'] = set['LandSlope'].map(LS)
 set['Neighborhood'] = set['Neighborhood'].map(N)
 set['Condition1'] = set['Condition1'].map(C1)
 set['BldgType'] = set['BldgType'].map(BT)
 set['HouseStyle'] = set['HouseStyle'].map(Paved)
 set['RoofStyle'] = set['RoofStyle'].map(rest)
 set['Exterior1st'] = set['Exterior1st'].map(rest)
 set['Exterior2nd'] = set['Exterior2nd'].map(EX2)
 set['MasVnrType'] = set['MasVnrType'].map(MSV)
 set['Foundation'] = set['Foundation'].map(F)
 set['Electrical'] = set['Electrical'].map(rest)
 set['Functional'] = set['Functional'].map(rest)
 set['GarageType'] = set['GarageType'].map(rest)
 set['GarageFinish'] = set['GarageFinish'].map(rest)
 set['SaleType'] = set['SaleType'].map(rest)
 set['SaleCondition'] = set['SaleCondition'].map(rest)
 col1 = ['MSZoning','BldgType','Exterior2nd']
for set in dataset1:
for c in col1:
set[c] = set[c].fillna(set[c].median() )
NFOLD = 2
kf = KFold(n_splits=NFOLD )<find_best_model_class>
log_price = np.log1p(train1['SalePrice'])
 targets = log_price
col = ['Id','SalePrice','Electrical_Mix', 'GarageQual_Ex', 'Exterior1st_Stone', 'Exterior1st_ImStucc', 'Exterior2nd_Other', 'HouseStyle_2.5Fin']
drop = ['Id','SalePrice']
train1.drop(drop,axis=1,inplace=True)
 test1.drop(['Id'],axis=1,inplace=True)
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=1000,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD 
<compute_test_metric>
train1['MoSold'].value_counts()
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<train_model>
numeric_feats = train1.dtypes[train1.dtypes != "object"].index
skewed_feats = train1[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewed_feats
NFOLD = 2
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<train_model>
cols = ['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',
'OverallCond', 'YearBuilt', 'MasVnrArea','TotalBsmtSF', 'BsmtFinSF1', 'GarageYrBlt', 'YearRemodAdd', 'GarageArea',
'BsmtFinSF2', 'BsmtUnfSF', '1stFlrSF',
'2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath',
'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr',
'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces',
'GarageCars', 'WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
'House_Age','YrBltAndRemod','TotalSF','Total_sqr_footage','Total_porch_sf','YrSold','MoSold']
scaler = preprocessing.StandardScaler()
scaler1 = preprocessing.RobustScaler()
scaler.fit(train1)
 train1 = scaler.transform(train1)
 test1 = scaler.transform(test1 )
NFOLD = 2
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<train_model>
rf = RandomForestRegressor(bootstrap=True,criterion='mse',random_state=42,max_depth=35,
max_features=35,n_estimators=2500,n_jobs=-1)
 xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=6, min_child_weight=4, n_estimators=4500,
n_jobs=4, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 lgb = LGBMRegressor(max_depth=25,num_leaves=120,learning_rate=0.01,n_jobs=-1,boosting_type='gbdt',
objective='regression',mertic='rmse', verbosity=1,bagging_fraction=0.7,
feature_fraction=0.5,bagging_frequency=6,bagging_seed=42,seed=42,
colsample_bylevel=1,colsample_bynode=1, colsample_bytree=1)
 knn = KNeighborsRegressor(n_neighbors=2,leaf_size=2,p=1,metric='manhattan',weights='distance')
 lasso = LassoCV(alphas=[5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008], max_iter=50000)
 ridge = make_pipeline(scaler1,RidgeCV(alphas=[14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5],cv=5))
svr = make_pipeline(scaler1, SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.01, max_depth=4,
max_features='sqrt', min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 dtr = DecisionTreeRegressor(criterion='mse',random_state=42,max_depth=35,
max_features='sqrt', min_samples_leaf=15, min_samples_split=10)
 abr = AdaBoostRegressor(dtr,n_estimators=4500,random_state=42,learning_rate=0.01)
 cat = CatBoostRegressor(depth=6,learning_rate=0.1,n_estimators=2500,eval_metric = 'RMSE' )
NFOLD = 4
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<train_model>
def get_stack() :
level0 = list()
level0.append(( 'rf',rf))
level0.append(( 'xgb',xgb))
level0.append(( 'knn',knn))
level0.append(( 'lgb',lgb))
level0.append(( 'svr',svr))
level1 = LinearRegression()
model = StackingRegressor(estimators=level0, final_estimator=level1, cv=5)
 return model
def get_models() :
models = {'rf':rf,'xgb':xgb,'knn':knn,'lgb':lgb,'svr':svr,'Stacking':get_stack() }
return models
def evaluate_model(model, X, y):
cv = RepeatedKFold(n_splits=5, n_repeats=3, random_state=1)
 scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1, error_score='raise')
 return scores
NFOLD = 10
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
svr.fit(train1,targets)
 rf.fit(train1,targets)
 xgb.fit(train1,targets)
 gbr.fit(train1,targets)
 abr.fit(train1,targets)
 lgb.fit(train1,targets)
 knn.fit(train1,targets)
 cat.fit(train1,targets )
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
VR = VotingRegressor([('rf',rf),('xgb',xgb),('knn',knn),('lgb',lgb),('gbr',gbr),('abr',abr),('cat',cat),('svr',svr)],n_jobs=-1)
 VR.fit(train1,targets)
 sqrt(mean_squared_error(targets, VR.predict(train1)) )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
level0 = [('rf',rf),('xgb',xgb),('lgb',lgb),('knn',knn),('gbr',gbr),('abr',abr),('cat',cat),('svr',svr),('vr',VR)]
level1 = LinearRegression()
stack = StackingRegressor(estimators=level0,final_estimator=level1, cv=5,n_jobs=-1)
 stack.fit(train1,targets)
 sqrt(mean_squared_error(targets, stack.predict(train1)) )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
stack_pred = np.expm1(stack.predict(test1))
vr_pred = np.expm1(VR.predict(test1))
rf_pred = np.expm1(rf.predict(test1))
xgb_pred = np.expm1(xgb.predict(test1))
lgb_pred = np.expm1(lgb.predict(test1))
cat_pred = np.expm1(cat.predict(test1))
svr_pred = np.expm1(svr.predict(test1))
abr_pred = np.expm1(abr.predict(test1))
gbr_pred = np.expm1(gbr.predict(test1))
knn_pred = np.expm1(knn.predict(test1))
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": stack_pred
})
 submission.to_csv('submission.csv', index=False )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
t = [105000.0,164370.59375,199657.75,201696.609375,187367.46875]
t1 = [122242.0,163769.0,184763.0,199176.0,191727.0]
m_new = 8945.36427302845
m_again = submission.iloc[0:5,1]
sqrt(mean_squared_error(t, m_again))