Kaggle competition: House Prices - Advanced Regression Techniques
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
train_all = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test_all = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
train_all.corr() ['SalePrice'].sort_values(ascending=False)[1:]
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
train_all['SalePrice'] = np.log1p(train_all['SalePrice'])
 cont_plot(train_all, 'SalePrice' )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
all_dataset = pd.concat(( train_all, test_all)).reset_index(drop = True)
 all_dataset.drop(['SalePrice'], axis = 1, inplace = True)
 all_dataset.shape
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
none_features = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
for feature in none_features:
all_dataset[feature] = all_dataset[feature].fillna('None' )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
print('If the BsmtFinType1 is {}, then the BsmtFinSF1 should be 0 instead of {}'.format(
(all_dataset[all_dataset['BsmtFinSF1'].isnull() == True]['BsmtFinType1'].iloc[0]),
all_dataset[all_dataset['BsmtFinSF1'].isnull() == True]['BsmtFinSF1'].iloc[0]))
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
zero_features =['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for feature in zero_features:
all_dataset[feature] = all_dataset[feature].fillna(0 )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<import_modules>
all_dataset['LotFrontage'] = all_dataset.groupby('LotShape')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
from sklearn.linear_model import Ridge, ElasticNet<compute_test_metric>
all_dataset['MSSubClass'] = all_dataset['MSSubClass'].astype(str)
 all_dataset['MSZoning'] = all_dataset.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
def metric(trueFVC, predFVC, predSTD):
clipSTD = np.clip(predSTD, 70 , 9e9)
 deltaFVC = np.clip(np.abs(trueFVC-predFVC), 0 , 1000)
 return np.mean(-1*(np.sqrt(2)*deltaFVC/clipSTD)- np.log(np.sqrt(2)*clipSTD)) 
<compute_test_metric>
null_cat = ['Utilities', 'Functional', 'SaleType',
'Exterior1st', 'Electrical', 'KitchenQual',
'Exterior2nd']
for feature in null_cat:
all_dataset[feature] = all_dataset[feature].fillna(statistics.mode(all_dataset[feature]))
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def kloss(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 1]
fvc_pred = y_pred[:, 0]
sigma_clip = sigma + C1
delta = tf.abs(y_true[:, 0] - fvc_pred)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def kmae(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 spread = tf.abs(( y_true[:, 0] - y_pred[:, 0])/(y_pred[:, 0] + 1.))
 return K.mean(spread)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * kloss(y_true, y_pred)+(1 - _lambda)*kmae(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 preds = L.Dense(2, activation="relu", name="preds" )(x)
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.99),
optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0.01, amsgrad=False),
metrics=[kloss])
 return model<prepare_x_and_y>
features_to_object = ['YrSold', 'MoSold']
for feature in features_to_object:
all_dataset[feature] = all_dataset[feature].astype(str )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values<choose_model_class>
def box_transform(df):
numerical_features = df.dtypes[df.dtypes != 'object'].index.drop('Id')
 for feature in numerical_features:
df[feature] = scipy.special.boxcox1p(df[feature], scipy.stats.boxcox_normmax(df[feature] + 1))
NFOLD = 5
kf = KFold(n_splits=NFOLD )<train_model>
final = pd.get_dummies(all_dataset ).reset_index(drop=True ).drop('Id', axis=1)
pe0 = np.zeros(( ze.shape[0], 2))
pred0 = np.zeros(( z.shape[0], 2))
pe1 = np.zeros(( ze.shape[0], 2))
pred1 = np.zeros(( z.shape[0], 2))
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 print("===================== NEURAL NET =============================")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=200, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=500))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=500))
print("predict val...")
 pred0[val_idx] = net.predict(z[val_idx], batch_size=500, verbose=0)
 print("predict test...")
 pe0 += net.predict(ze, batch_size=500, verbose=0)/ NFOLD
print("===================== RIDGE REG =============================")
 clf = Ridge(alpha=0.05)
 clf.fit(z[tr_idx], y[tr_idx])
 pred1[val_idx, 0] = clf.predict(z[val_idx])
 pred_std = np.mean(np.abs(y[val_idx] - pred1[val_idx, 0])) * np.sqrt(2)
 pred1[val_idx, 1] = pred_std
print("val", metric(y[val_idx], pred1[val_idx, 0], pred1[val_idx, 1]))
pe1[:, 0] += clf.predict(ze)/ NFOLD
pe1[:, 1] += pred_std / NFOLD
pred0[:, 1] = pred0[:, 1] + 70 .<prepare_output>
X_train = final.iloc[:len(train_all),:]
X_test = final.iloc[len(train_all):,:]
Y_train = train_all['SalePrice']
w = 0.5
pred =(1-w)* pred0 + w * pred1
pe =(1-w)* pe0 + w * pe1
pe[:, 1] = pe1[:, 1]
pred[:, 1] = pred1[:, 1]<compute_test_metric>
X_sub_train, X_sub_test, Y_sub_train, Y_sub_test = train_test_split(X_train, Y_train, test_size = 0.3, random_state=42)
 print(X_sub_train.shape, Y_sub_train.shape)
 print(X_sub_test.shape, Y_sub_test.shape )
print("oof neural net", metric(y, pred0[:, 0], pred0[:, 1]))
print("oof ridge", metric(y, pred1[:, 0], pred1[:, 1]))
print("oof ensemble", metric(y, pred[:, 0], pred[:, 1]))<compute_test_metric>
ridge = Ridge(normalize=True)
 ridge_params = {'alpha':np.linspace(0.1,1,10),'tol':np.linspace(0.001,0.01,10)}
ridge_search = GridSearchCV(ridge, ridge_params, cv=5)
 ridge_search.fit(X_train, Y_train )
sigma_opt = mean_absolute_error(y, pred[:, 0])
 sigma_mean = np.mean(pred[:, 1])
 print(sigma_opt, sigma_mean )<prepare_output>
alpha_ridge = list(ridge_search.best_params_.values())[0]
tol_ridge = list(ridge_search.best_params_.values())[1]
ridge_test = Ridge(alpha = alpha_ridge, tol = tol_ridge, normalize=True)
 ridge_test.fit(X_sub_train, Y_sub_train)
 Y_fit_ridge = ridge_test.predict(X_sub_test)
 rmse_ridge = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_ridge))
rmse_ridge
sub['FVC1'] = pe[:, 0]
sub['Confidence1'] = pe[:, 1]<prepare_output>
lasso = Lasso(normalize=True)
 lasso_params = {'alpha':np.linspace(0.00001,0.001,100)}
lasso_search = GridSearchCV(lasso, lasso_params, cv=5)
 lasso_search.fit(X_train, Y_train )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
alpha_lasso = list(lasso_search.best_params_.values())[0]
lasso_test = Lasso(alpha=alpha_lasso, normalize=True)
 lasso_test.fit(X_sub_train, Y_sub_train)
 Y_fit_lasso = lasso_test.predict(X_sub_test)
 rmse_lasso = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_lasso))
rmse_lasso
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
lgbm = LGBMRegressor()
lgbm_params = {'num_leaves':[5,10,20], 'max_depth':[3,5,8]
,'learning_rate':[0.1,0.2,0.3], 'n_estimators':[75,100,125]}
lgbm_search = GridSearchCV(lgbm, lgbm_params, cv=5)
 lgbm_search.fit(X_train, Y_train )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1 
<save_to_csv>
learning_rate_lgbm = list(lgbm_search.best_params_.values())[0]
max_depth_lgbm = list(lgbm_search.best_params_.values())[1]
n_estimators_lgbm = list(lgbm_search.best_params_.values())[2]
num_leaves_lgbm = list(lgbm_search.best_params_.values())[3]
lgbm_test = LGBMRegressor(learning_rate = learning_rate_lgbm, max_depth = max_depth_lgbm,
n_estimators = n_estimators_lgbm, num_leaves= num_leaves_lgbm)
 lgbm_test.fit(X_sub_train, Y_sub_train)
 Y_fit_lgbm = lgbm_test.predict(X_sub_test)
 rmse_lgbm = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_lgbm))
rmse_lgbm
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<set_options>
elastic = ElasticNet()
elastic_params = {'alpha':np.linspace(0.005,0.01,5), 'l1_ratio':[0.01,0.05]}
elastic_search = GridSearchCV(elastic, elastic_params, cv=5)
 elastic_search.fit(X_train,Y_train )
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )<load_from_csv>
alpha_elastic = list(elastic_search.best_params_.values())[0]
l1_ratio_elastic = list(elastic_search.best_params_.values())[1]
elastic_test = ElasticNet(alpha = alpha_elastic
, l1_ratio = l1_ratio_elastic)
 elastic_test.fit(X_sub_train, Y_sub_train)
 Y_fit_elastic = elastic_test.predict(X_sub_test)
 rmse_elastic = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_elastic))
rmse_elastic
df_train=pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 df_test=pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<categorify>
def preprocess(text):
text=text.lower()
text = re.sub(r'https?:\/\/.*[\r
]*', '', text)
 text = re.sub(r'http?:\/\/.*[\r
]*', '', text)
 text=text.replace(r'&amp;?',r'and')
 text=text.replace(r'&lt;',r'<')
 text=text.replace(r'&gt;',r'>')
 text = re.sub(r"(?:\@)\w+", '', text)
 text=text.encode("ascii",errors="ignore" ).decode()
text=re.sub(r'[:"
text=re.sub(r'[!]+','!',text)
 text=re.sub(r'[?]+','?',text)
 text=re.sub(r'[.]+','.',text)
 text=re.sub(r"'","",text)
 text=re.sub(r"\(","",text)
 text=re.sub(r"\)","",text)
 text=" ".join(text.split())
 return text
df_train['text'] = df_train['text'].apply(preprocess)
 df_test['text'] = df_test['text'].apply(preprocess)
 df_train=df_train[df_train["text"]!='']<count_values>
xgboost_test = XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 xgboost_test.fit(X_sub_train, Y_sub_train)
 Y_fit_xgb = xgboost_test.predict(X_sub_test)
 rmse_xgb = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_xgb))
rmse_xgb
df_train["target"].value_counts()<define_variables>
estimators = [('ridge', ridge_test), 
('lasso', lasso_test), 
('lgbm', lgbm_test), 
('elastic', elastic_test), 
('xgb', xgboost_test)]
stack = StackingRegressor(estimators = estimators)
 stack.fit(X_sub_train,Y_sub_train)
 Y_fit_stack = stack.predict(X_sub_test)
 rmse_stack = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_stack))
rmse_stack
<load_pretrained><EOS> 
stack_final = StackingRegressor(estimators = estimators)
 stack_final.fit(X_train, Y_train)
 Y_fit_final = stack_final.predict(X_test)
 pred = np.expm1(Y_fit_final)
 submission = pd.DataFrame({'Id':test_all['Id'], 'SalePrice':pred})
 submission.to_csv("submission.csv", index=False )