Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
warnings.filterwarnings('ignore' )
path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')
 assert path.exists()<load_from_csv>
train = pd.read_csv('.. /input/train.csv',index_col = 'Id')
 test = pd.read_csv('.. /input/test.csv',index_col = 'Id' )
TRAIN_TYPES={"Patient": "category",
"Weeks": "int16", "FVC": "int32", 'Percent': 'float32', "Age": "uint8",
"Sex": "category", "SmokingStatus": "category" }
SUBMISSION_TYPES={"Patient_Week": "category", "FVC": "int32", "Confidence": "int16"}
def read_data(path):
train_df = pd.read_csv(path/'train.csv', dtype = TRAIN_TYPES)
 test_df = pd.read_csv(path/'test.csv', dtype = TRAIN_TYPES)
 submission_df = pd.read_csv(path/'sample_submission.csv', dtype = SUBMISSION_TYPES)
 train_df.drop_duplicates(keep='first', inplace=True, subset=['Patient','Weeks'])
 return train_df, test_df, submission_df<load_from_disk>
train.corr() ['SalePrice'].sort_values(ascending=False )
train_df, test_df, submission_df = read_data(path )<merge>
features = ['OverallQual','YearBuilt','YearRemodAdd','TotalBsmtSF','1stFlrSF',
'GrLivArea','FullBath','TotRmsAbvGrd','GarageCars','GarageArea']
def prepare_submission(df, test_df):
df['Patient'] = df['Patient_Week'].apply(lambda x:x.split('_')[0])
 df['Weeks'] = df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
df = df[['Patient','Weeks','Confidence','Patient_Week']]
df = df.merge(test_df.drop('Weeks', axis=1 ).copy() , on=['Patient'])
 return df<categorify>
train['log_SalePrice']=np.log(train['SalePrice']+1)
 saleprices=train[['SalePrice','log_SalePrice']]
saleprices.head(5 )
submission_df = prepare_submission(submission_df, test_df )<feature_engineering>
train=train.drop(columns=['SalePrice','log_SalePrice'] )
def adapt_percent_in_submission() :
previous_match = None
for i, r in submission_df.iterrows() :
in_training = train_df[(train_df['Patient'] == r['Patient'])&(train_df['Weeks'] == r['Weeks'])]
if(len(in_training)> 0):
previous_match = in_training['Percent'].item()
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match
elif previous_match is not None:
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match<concatenate>
all_data = pd.concat(( train, test))
print(all_data.shape)
 all_data.head()
train_df['WHERE'] = 'train'
test_df['WHERE'] = 'val'
submission_df['WHERE'] = 'test'
data = train_df.append([test_df, submission_df] )<feature_engineering>
null_data = pd.DataFrame(all_data.isnull().sum().sort_values(ascending=False))
null_data.columns = ['Null Count']
null_data.index.name = 'Feature'
null_data
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<filter>
( null_data/len(all_data)) * 100
base = data.loc[data.Weeks == data.min_week]<categorify>
for col in('PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType', 'MSSubClass'):
all_data[col] = all_data[col].fillna('None' )
base = base[['Patient','FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_Percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base<merge>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath',
'BsmtHalfBath', 'MasVnrArea'):
all_data[col] = all_data[col].fillna(0 )
data = data.merge(base, on='Patient', how='left' )<feature_engineering>
for col in('MSZoning', 'Electrical', 'KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType', 'Functional', 'Utilities'):
all_data[col] = all_data[col].fillna(all_data[col].mode() [0] )
data['base_week'] = data['Weeks'] - data['min_week']
data['base_week'] = data['base_week']
del base<filter>
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
data[data['Patient'] == 'ID00421637202311550012437']<data_type_conversions>
all_data=all_data.drop(columns=['Street','Utilities','Condition2','RoofMatl',
'Heating','PoolArea','PoolQC','MiscVal','MiscFeature'] )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<rename_columns>
all_data['MSSubClass']=all_data['MSSubClass'].astype(str)
 all_data['MoSold']=all_data['MoSold'].astype(str)
 all_data['YrSold']=all_data['YrSold'].astype(str )
data = data.rename(columns={"Age": "age", "min_FVC": "BASE", "base_week": "week", "Percent": "percent"})
 FE += ['age','week','BASE', 'percent']
FE<filter>
all_data['NoLowQual']=(all_data['LowQualFinSF']==0)
 all_data['NoOpenPorch']=(all_data['OpenPorchSF']==0)
 all_data['NoWoodDeck']=(all_data['WoodDeckSF']==0)
 all_data['NoGarage']=(all_data['GarageArea']==0 )
train_df = data.loc[data.WHERE=='train']
test_df = data.loc[data.WHERE=='val']
submission_df = data.loc[data.WHERE=='test']
del data<sort_values>
Bsmt=Bsmt.replace(to_replace='Po', value=1)
 Bsmt=Bsmt.replace(to_replace='Fa', value=2)
 Bsmt=Bsmt.replace(to_replace='TA', value=3)
 Bsmt=Bsmt.replace(to_replace='Gd', value=4)
 Bsmt=Bsmt.replace(to_replace='Ex', value=5)
 Bsmt=Bsmt.replace(to_replace='None', value=0)
 Bsmt=Bsmt.replace(to_replace='No', value=1)
 Bsmt=Bsmt.replace(to_replace='Mn', value=2)
 Bsmt=Bsmt.replace(to_replace='Av', value=3)
 Bsmt=Bsmt.replace(to_replace='Gd', value=4)
 Bsmt=Bsmt.replace(to_replace='Unf', value=1)
 Bsmt=Bsmt.replace(to_replace='LwQ', value=2)
 Bsmt=Bsmt.replace(to_replace='Rec', value=3)
 Bsmt=Bsmt.replace(to_replace='BLQ', value=4)
 Bsmt=Bsmt.replace(to_replace='ALQ', value=5)
 Bsmt=Bsmt.replace(to_replace='GLQ', value=6 )
train_df.sort_values(['Patient', 'Weeks'], inplace=True )<prepare_x_and_y>
Bsmt['BsmtScore']= Bsmt['BsmtQual'] * Bsmt['BsmtCond'] * Bsmt['TotalBsmtSF']
all_data['BsmtScore']=Bsmt['BsmtScore']
y = train_df['FVC']
y<define_variables>
Bsmt['BsmtFin'] =(Bsmt['BsmtFinSF1'] * Bsmt['BsmtFinType1'])+(Bsmt['BsmtFinSF2'] * Bsmt['BsmtFinType2'])
 all_data['BsmtFinScore']=Bsmt['BsmtFin']
all_data['BsmtDNF']=(all_data['BsmtFinScore']==0 )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )<define_variables>
lot=['LotFrontage', 'LotArea','LotConfig','LotShape']
Lot=all_data[lot]
Lot.head()
seed_everything(42 )<compute_test_metric>
garage=['GarageArea','GarageCars','GarageCond','GarageFinish','GarageQual','GarageType','GarageYrBlt']
Garage=all_data[garage]
Garage=Garage.replace(to_replace='Po', value=1)
 Garage=Garage.replace(to_replace='Fa', value=2)
 Garage=Garage.replace(to_replace='TA', value=3)
 Garage=Garage.replace(to_replace='Gd', value=4)
 Garage=Garage.replace(to_replace='Ex', value=5)
 Garage=Garage.replace(to_replace='None', value=0)
 Garage=Garage.replace(to_replace='Unf', value=1)
 Garage=Garage.replace(to_replace='RFn', value=2)
 Garage=Garage.replace(to_replace='Fin', value=3)
 Garage=Garage.replace(to_replace='CarPort', value=1)
 Garage=Garage.replace(to_replace='Basment', value=4)
 Garage=Garage.replace(to_replace='Detchd', value=2)
 Garage=Garage.replace(to_replace='2Types', value=3)
 Garage=Garage.replace(to_replace='Basement', value=5)
 Garage=Garage.replace(to_replace='Attchd', value=6)
 Garage=Garage.replace(to_replace='BuiltIn', value=7)
 Garage.head()
C1_val = 70
C2_val = 1000
C1, C2 = C1_val, C2_val
q = np.array([0.2, 0.50, 0.8])
 def score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = np.max(sigma)
 delta = np.abs(y_true[:, 0] - fvc_pred)
 delta = np.min(delta)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 def qloss(y_true, y_pred):
print('y_true.shape', y_true.shape)
 print('y_pred.shape', y_pred.shape)
 e = y_true - y_pred
v = np.max(q*e,(q-1)*e)
 return np.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
y_true = y_true.reshape(-1, 1)
 return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def tilted_loss(q, y, f):
e =(y - f)
 return keras.backend.mean(keras.backend.maximum(q * e,(q - 1)* e), axis=-1 )<init_hyperparams>
def onehot(col_list):
global all_data
while len(col_list)!=0:
col=col_list.pop(0)
 data_encoded=pd.get_dummies(all_data[col], prefix=col)
 all_data=pd.merge(all_data, data_encoded, on='Id')
 all_data=all_data.drop(columns=col)
 print(all_data.shape )
n_estimators = 30000
if not 'sub_row' in locals() :
sub_row = 0.75
if not 'bagging_freq' in locals() :
bagging_freq = 1
if not 'learning_rate' in locals() :
learning_rate = 0.4
leave_size = 4
lgb_params = {
"objective": 'quantile',
'n_jobs': 1,
'max_depth': leave_size + 1,
'num_leaves': 2**leave_size-1,
"min_data_in_leaf": 2**(leave_size + 1)-1,
"n_estimators": n_estimators,
'learning_rate': 8e-3,
'colsample_bytree': 0.9,
'boosting_type': 'gbdt',
"early_stopping_rounds": 100,
'verbosity': 1000,
"metric": ["rmse", "mse"]
}
cat_feats = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']<choose_model_class>
onehot(list(non_numeric))
optimizer = tf.optimizers.Adam(learning_rate=0.001)
 early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)
 EPOCHS = 200
BATCH_SIZE = 32
UNITS = 75<train_on_grid>
def log_transform(col_list):
transformed_col=[]
while len(col_list)!=0:
col=col_list.pop(0)
 if all_data[col].skew() > 0.5:
all_data[col]=np.log(all_data[col]+1)
 transformed_col.append(col)
 else:
pass
print(f"{len(transformed_col)} features had been tranformed")
 print(all_data.shape )
def keras_pred(x_train, train_labels, q):
print(q)
 train_labels = np.array(train_labels.values, dtype='float32')
 input_dim = x_train.shape[1]
model = keras.Sequential([
keras.layers.Dense(UNITS, input_dim=input_dim),
keras.layers.Activation('relu'),
keras.layers.Dense(UNITS, activation=tf.nn.relu),
keras.layers.Dense(1)
 ])
 model.compile(loss=lambda y, f: tilted_loss(q, y, f), optimizer=optimizer)
 model.fit(x_train, train_labels, epochs=EPOCHS, batch_size=BATCH_SIZE,
verbose=1, validation_split=0.2, callbacks=[early_stop])
 return model<predict_on_test>
numeric=all_data.select_dtypes(include=np.number)
 log_transform(list(numeric))
def ols_quantile(m, X, q):
mean_pred = m.predict(X)
 se = np.sqrt(m.scale)
 return mean_pred + norm.ppf(q)* se<define_search_space>
train=all_data[:len(train)]
test=all_data[len(train):]
ensemble_weights = [4./11, 1./11, 6./11]
assert 1.0 - np.sum(ensemble_weights)< 0.00001<define_variables>
from sklearn.linear_model import ElasticNet, Lasso
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import cross_val_score
from sklearn.metrics import make_scorer
from sklearn import linear_model, model_selection, ensemble, preprocessing
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet,SGDRegressor
from sklearn.ensemble import RandomForestRegressor,BaggingRegressor, GradientBoostingRegressor, AdaBoostRegressor
from sklearn.model_selection import GridSearchCV, KFold, cross_val_score
from sklearn.neighbors import KNeighborsRegressor
import xgboost as xgb
import lightgbm as lgb
NFOLD = 5
kf = KFold(n_splits=NFOLD, shuffle=False)
 pred = {a: np.zeros(( train_df.shape[0])) for a in q.tolist() }<train_model>
def rmse(predict, actual):
predict = np.array(predict)
 actual = np.array(actual)
 distance = predict - actual
square_distance = distance ** 2
mean_square_distance = square_distance.mean()
score = np.sqrt(mean_square_distance)
 return score
rmse_score = make_scorer(rmse)
 rmse_score
models = []
linear_models = []
keras_models = []
for cnt,(tr_idx, val_idx)in tqdm(enumerate(kf.split(X)) , total=NFOLD):
X_train, y_train = X.loc[tr_idx], y.loc[tr_idx]
X_valid, y_valid = X.loc[val_idx], y.loc[val_idx]
print(f"FOLD {cnt}", X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)
 for qi, quantile_alpha in enumerate(q.tolist()):
lgb_params['alpha'] = quantile_alpha
m_lgb_regressor = lgb.LGBMRegressor(**lgb_params)
 m_lgb_regressor.fit(X=X_train, y=y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_names=['train mloss', 'valid mloss'],
eval_metric=lgb_params['metric'],
verbose=lgb_params['verbosity'],
early_stopping_rounds=lgb_params["early_stopping_rounds"],
categorical_feature=cat_feats)
 lin_model = sm.RLM(y_train, X_train, M=sm.robust.norms.AndrewWave() ).fit()
lin_predict = ols_quantile(lin_model, X_valid, quantile_alpha)
 keras_model = keras_pred(X_train, y_train, quantile_alpha)
 keras_predict = keras_model.predict(X_valid ).reshape(-1)
 lgb_predict = m_lgb_regressor.predict(X_valid)
 pred[quantile_alpha][val_idx] = np.average([lin_predict, lgb_predict, keras_predict], axis = 0, weights=ensemble_weights)
 models.append(m_lgb_regressor)
 linear_models.append(lin_model)
 keras_models.append(keras_model )<compute_test_metric>
feature_names=list(all_data)
 X_train = train[feature_names]
X_test = test[feature_names]
y_train = saleprices['log_SalePrice']
full_preds = np.vstack([pred[a] for a in q] ).T
score(np.array(y ).reshape(-1, 1), full_preds )<predict_on_test>
def score(model):
score = cross_val_score(model, X_train, y_train, cv=5, scoring=rmse_score ).mean()
return score
pred = []
for i in range(NFOLD):
cur_pred = []
for j, quantile in enumerate(q):
model_idx = i * 3 + j
model = models[model_idx]
lin_predict = ols_quantile(linear_models[model_idx], submission_df[FE], quantile)
 dbmc_predict = model.predict(submission_df[FE])
 keras_predict = keras_models[model_idx].predict(submission_df[FE] ).reshape(-1)
 cur_pred.append(np.average([lin_predict, dbmc_predict, keras_predict], axis = 0, weights=ensemble_weights))
pred.append(np.array(cur_pred ).T )<prepare_output>
YouTubeVideo('EuBBz3bI-aA',width=700, height=350 )
submission_df['FVC1'] = final_preds[:,1]
submission_df['Confidence1'] = final_preds[:, 2] - final_preds[:, 0]<save_to_csv>
model_Lasso= make_pipeline(RobustScaler() , Lasso(alpha =0.000327, random_state=18))
model_ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.00052, l1_ratio=0.70654, random_state=18))
model_GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =18)
 model_XGB=xgb.XGBRegressor(n_jobs=-1, n_estimators=849, learning_rate=0.015876,
max_depth=58, colsample_bytree=0.599653, colsample_bylevel=0.287441, subsample=0.154134, seed=18)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 forest_reg = RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=None,
max_features=60, max_leaf_nodes=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
n_estimators=70, n_jobs=1, oob_score=False, random_state=42,
verbose=0, warm_start=False)
submission_df[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<load_from_csv>
model_Lasso.fit(X_train, y_train)
 Lasso_Predictions=np.exp(model_Lasso.predict(X_test)) -1
model_ENet.fit(X_train, y_train)
 ENet_Predictions=np.exp(model_ENet.predict(X_test)) -1
model_XGB.fit(X_train, y_train)
 XGB_Predictions=np.exp(model_XGB.predict(X_test)) -1
model_GBoost.fit(X_train, y_train)
 GBoost_Predictions=np.exp(model_GBoost.predict(X_test)) -1
model_lgb.fit(X_train, y_train)
 lgb_Predictions=np.exp(model_lgb.predict(X_test)) -1
forest_reg.fit(X_train, y_train)
 forest_reg_Predictions=np.exp(forest_reg.predict(X_test)) -1
submission_final_df = pd.read_csv("submission.csv" )<import_modules>
scores ={}
scores.update({'Lasso':score(model_Lasso)})
 scores.update({"Elastic Net":score(model_ENet)})
 scores.update({"XGB":score(model_XGB)})
 scores.update({"Gradient Boost":score(model_GBoost)})
 scores.update({"lgb":score(model_lgb)})
 scores.update({"Random Forest":score(forest_reg)} )
import tensorflow as tf
import pandas as pd
import numpy as np
import os
import random
import pickle
import time
from tensorflow import keras as K
from tensorflow.keras import layers as L
from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error
import matplotlib.pyplot as plt<set_options>
scores_df =pd.DataFrame(list(scores.items()),columns=['Model','Score'])
 scores_df.sort_values(['Score'] )
def seed_all(seed = 20):
os.environ['PYTHONHASHSEED']=str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_all(20 )<load_from_csv>
Image(url= "https://blogs.sas.com/content/subconsciousmusings/files/2017/05/weighted-unweighted.png" )
train = pd.read_csv('/kaggle/input/data-preparation-for-osic/train.csv')
 raw_test = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')
 X_prediction = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<define_variables>
ensemble =(Lasso_Predictions + XGB_Predictions + lgb_Predictions + ENet_Predictions +
forest_reg_Predictions)/5
ensemble
ID='Patient_Week'
PINBALL_QUANTILE = [0.255, 0.50, 0.745]
LAMBDA_LOSS = 0.585
EPOCH = [54, 55, 20, 60, 23]
BATCH_SIZE = 128
NFOLD = 5<compute_test_metric>
submission=pd.read_csv('.. /input/sample_submission.csv')
 submission['SalePrice']= ensemble
<compute_test_metric><EOS> 
submission.to_csv('submission.csv',index=False )