Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
sns.set_style('whitegrid')
 warnings.simplefilter(action='ignore')
test = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})
 submission6 = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 submission6['Patient'] = submission6['Patient_Week'].apply(lambda x: x.split('_')[0])
 submission6['predict_Week'] = submission6['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 test = submission6.drop(columns=['FVC', 'Confidence'] ).merge(test, on='Patient')
 test['Week_passed'] = test['predict_Week'] - test['base_Week']
test.set_index('Patient_Week', inplace=True )<categorify>
a = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 b = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
folds = train[['Patient', TARGET]].copy()
Fold = GroupKFold(n_splits=N_FOLD)
 groups = folds['Patient'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[TARGET], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int )<prepare_x_and_y>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):
trn_idx = folds[folds.fold!=fold_num].index
val_idx = folds[folds.fold==fold_num].index
y_tr = target.iloc[trn_idx].values
X_tr = train_df.iloc[trn_idx][features].values
y_val = target.iloc[val_idx].values
X_val = train_df.iloc[val_idx][features].values
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
clf.fit(X_tr, y_tr)
 oof[val_idx] = clf.predict(X_val)
 predictions += clf.predict(test_df[features])
 return oof, predictions
def run_kfold_model(clf, train, test, folds, features, target, n_fold=5):
oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_ in range(n_fold):
_oof, _predictions = run_single_model(clf,
train,
test,
folds,
features,
target,
fold_num=fold_)
 oof += _oof
predictions += _predictions/n_fold
return oof, predictions<categorify>
print('The shape of our training set: %s houses and %s features'%(a.shape[0],a.shape[1]))
print('The shape of our training set: %s houses and %s features'%(b.shape[0],b.shape[1]))
print('The testing set has 1 feature less than the training set, which is SalePrice, the target to predict ' )
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']
features = [c for c in features if c not in drop_features]
if cat_features:
ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')
 ce_oe.fit(train)
 train = ce_oe.transform(train)
 test = ce_oe.transform(test )<categorify>
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"Total size is {c1.shape}" )
clf = Ridge(alpha=0.1)
 oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)
 train['FVC_pred'] = oof
test['FVC_pred'] = predictions<feature_engineering>
msv1(c1, 20, color=sns.color_palette('Reds',15))
train['Confidence'] = 100
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )<find_best_params>
c=c1.dropna(thresh=len(c1)*0.8, axis=1)
 print(f"We dropped {c1.shape[1]-c.shape[1]} features in the combined set" )
def loss_func(weight, row):
confidence = weight
sigma_clipped = max(confidence, 70)
 diff = abs(row['FVC'] - row['FVC_pred'])
 delta = min(diff, 1000)
 score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)
 return -score
results = []
tk0 = tqdm(train.iterrows() , total=len(train))
for _, row in tk0:
loss_partial = partial(loss_func, row=row)
 weight = [100]
result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')
 x = result['x']
results.append(x[0] )<feature_engineering>
NA=c[allna.index.to_list() ]
train['Confidence'] = results
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )<categorify>
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )
TARGET = 'Confidence'
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']
features = [c for c in features if c not in drop_features]
oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD )<feature_engineering>
ffill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data
train['Confidence'] = oof
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )<feature_engineering>
msv2(c )
test['Confidence'] = predictions
test = test.reset_index()<merge>
d=filling_NA(c, ffill_cols )
submission6 = submission6[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')
 submission6 = submission6.rename(columns={'FVC_pred': 'FVC'} )<feature_engineering>
fill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )
for i in range(len(otest)) :
submission6.loc[submission6['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
submission6.loc[submission6['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )
submission6.to_csv('submission_Ridge.csv', index=False, float_format='%.1f')
<drop_column>
for col in NAcols:
if c[col].dtype != "object":
c[col]= c[col].fillna(0 )
submission_final =(subm.drop(['Patient_Week'], axis=1)+ submission1.drop(['Patient_Week'], axis=1)+ submission2.drop(['Patient_Week'], axis=1)+ submission3.drop(['Patient_Week'], axis=1)+ submission4.drop(['Patient_Week'], axis=1)) /5 
<feature_engineering>
c['TotalArea'] = c['TotalBsmtSF'] + c['1stFlrSF'] + c['2ndFlrSF'] + c['GrLivArea'] +c['GarageArea']
c['Bathrooms'] = c['FullBath'] + c['HalfBath']*0.5
c['Year average']=(c['YearRemodAdd']+c['YearBuilt'])/2
submission_final['Patient_Week'] = subm['Patient_Week'] 
<drop_column>
c['MSSubClass'] = c['MSSubClass'].apply(str)
 c['YrSold'] = c['YrSold'].astype(str )
submission_final.drop(['FVC1','Confidence1'],axis=1,inplace=True )<merge>
cb=pd.get_dummies(c)
 print(f"the shape of the original dataset {c.shape}")
 print(f"the shape of the encoded dataset {cb.shape}")
 print(f"We have {cb.shape[1]- c.shape[1]} new encoded features" )
submission_final=pd.merge(submission_final,submission5,on='Patient_Week',how='inner' ).merge(submission6,on='Patient_Week',how='inner' ).rename(columns={'FVC':'FVC1','Confidence':'Confidence1'} )<feature_engineering>
Train = cb[:na]
Test = cb[na:]
submission_final['FVC']=(submission_final['FVC_x']+submission_final['FVC_y']+submission_final['FVC1'])/3
submission_final['Confidence'] =(submission_final['Confidence1']+submission_final['Confidence_x']+submission_final['Confidence_y'])/3<save_to_csv>
a['GrLivArea'].sort_values(ascending=False ).head(2 )
submission_final[['Patient_Week','FVC','Confidence']].to_csv("submission.csv",index=False )<set_options>
a['TotalBsmtSF'].sort_values(ascending=False ).head(1 )
pd.set_option('display.max_columns', 60)
 pd.set_option('display.max_rows', 100 )<define_variables>
a['MasVnrArea'].sort_values(ascending=False ).head(1 )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<load_from_csv>
a['1stFlrSF'].sort_values(ascending=False ).head(1 )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<count_unique_values>
a['TotRmsAbvGrd'].sort_values(ascending=False ).head(1 )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
train=Train[(Train['GrLivArea'] < 4600)&(Train['MasVnrArea'] < 1500)]
print(f'We removed {Train.shape[0]- train.shape[0]} outliers' )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','FVC', 'Percent']] 
.rename({'FVC': 'base_FVC', 'Percent':'base_Percent'}, axis=1)
.groupby('Patient')
.first() 
.reset_index()
)<merge>
target=a[['SalePrice']]
target.loc[1298]
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<categorify>
target.loc[523]
FE = list(data.Sex.unique())+ list(data.SmokingStatus.unique())
 data = pd.concat([
data,
pd.get_dummies(data.Sex),
pd.get_dummies(data.SmokingStatus)
 ], axis=1 )<categorify>
pos = [1298,523, 297]
target.drop(target.index[pos], inplace=True )
def Normalization(df):
def get_fillness(series):
return(series - series.min())/(series.max() - series.min())
 df['Age'] = get_fillness(df['Age'])
 df['base_FVC'] = get_fillness(df['base_FVC'])
 df['base_week'] = get_fillness(df['base_week'])
 df['base_Percent'] = get_fillness(df['base_Percent'])
 return df
FE += ['Age','base_FVC','base_week','base_Percent']
data = Normalization(data )<filter>
print('We make sure that both train and target sets have the same row number after removing the outliers:')
 print('Train: ',train.shape[0], 'rows')
 print('Target:', target.shape[0],'rows' )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<import_modules>
from scipy.stats import skew
class Mish(Activation):
def __init__(self, activation, **kwargs):
super(Mish, self ).__init__(activation, **kwargs)
 self.__name__ = 'Mish'
def mish(inputs):
return inputs * tf.math.tanh(tf.math.softplus(inputs))
get_custom_objects().update({'Mish': Mish(mish)} )<compute_test_metric>
target["SalePrice"] = np.log1p(target["SalePrice"] )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="Mish", name="d1" )(z)
 x = L.Dense(100, activation="Mish", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="NN")
 model.compile(loss=mloss(0.8),
optimizer=tf.keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<compute_test_metric>
x=train
y=target.SalePrice.values
def calc_cv_score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(y_true[:, 0] - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return -np.mean(metric )<split>
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=42 )
%%time
cnt = 0
BATCH_SIZE =256
EPOCHS = 1500
NFOLD = 11
kf = GroupKFold(n_splits=NFOLD)
 y = tr['FVC'].values.astype('float32')
 z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
for tr_idx, val_idx in kf.split(z, y, tr['Patient']):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 es = tf.keras.callbacks.EarlyStopping(monitor='val_loss',patience=200, min_delta = 0.000001,
verbose=1, mode='min')
 lr_sch = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.4, patience=50,
verbose=0, mode='min', min_delta=0.000001, cooldown=0, min_lr=0)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
callbacks = [es, lr_sch],
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print(calc_cv_score(y[val_idx].reshape(-1, 1), pred[val_idx]))
print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print("CV SCORE", calc_cv_score(y.reshape(-1, 1), pred))<compute_test_metric>
scaler= RobustScaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test )
tr['FVC_pred'] = pred[:, 1]
tr['Confidence_pred'] = pred[:, 2] - pred[:, 0]
df_last_3 = tr.groupby('Patient' ).tail(3 ).reset_index(drop=True)
 X = df_last_3[['Weeks', 'FVC', 'FVC_pred', 'Confidence_pred']].values
C = 0
def calc_tunned_score(y_true, y_pred, Conf):
sigma = Conf
fvc_pred = y_pred
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(y_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip*sq2)
 return -np.mean(metric)
 def objective(trial, X, y):
a = trial.suggest_uniform('a', 0, 15)
 b = trial.suggest_uniform('b', -100, 100)
 y = a * X[:, 0] + b
New_Confidence = X[:, 3] + y
return calc_tunned_score(X[:, 1], X[:, 2], New_Confidence)
 n_trials = 500
obj = partial(objective, X=X, y=C)
 study = optuna.create_study(direction="maximize")
 optuna.logging.disable_default_handler()
study.optimize(obj, n_trials=n_trials )<train_model>
print('last 3 score befor tuning', calc_tunned_score(X[:, 1], X[:, 2], X[:, 3]))
print('last 3 score after tuning', study.best_value)
 param = {k:v for k,v in study.best_params.items() }
print('param', param)
 print('Training data score', calc_tunned_score(tr['FVC'].values, tr['FVC_pred'].values, tr['Confidence_pred'].values+param['a']*tr['Weeks'].values+param['b']))<compute_test_metric>
def score(y_pred):
return str(math.sqrt(sklm.mean_squared_error(y_test, y_pred)) )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
unc = unc +(param['a'] * tr['Weeks'] + param['b'])
 sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
ridge=Ridge()
parameters= {'alpha':[x for x in range(1,101)]}
ridge_reg=ms.GridSearchCV(ridge, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 ridge_reg.fit(x_train,y_train)
 print(f"The best value of Alpha is: {ridge_reg.best_params_}")
 print(f"The best score achieved with Alpha=11 is: {math.sqrt(-ridge_reg.best_score_)}")
 ridge_pred=math.sqrt(-ridge_reg.best_score_ )
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1', 'Weeks']].copy()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1'] + param['a'] * subm.loc[~subm.FVC1.isnull() ,'Weeks'] + param['b']<save_to_csv>
ridge_mod=Ridge(alpha=18)
 ridge_mod.fit(x_train,y_train)
 y_pred_train=ridge_mod.predict(x_train)
 y_pred_test=ridge_mod.predict(x_test)
 print(f'Root Mean Square Error train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_pred_train)))}')
 print(f'Root Mean Square Error test = {score(y_pred_test)}' )
otest = pd.read_csv(f"{ROOT}/test.csv")
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<install_modules>
Ridge_CV=Ridge(alpha=18)
 MSEs=ms.cross_val_score(Ridge_CV, x, y, scoring='neg_mean_squared_error', cv=5)
 print("RMSE scores of the 5 folds:")
 for i,j in enumerate(MSEs):
j= math.sqrt(np.mean(-j))
print(f'Fold {i}: {round(j,4)}')
 print(f'Mean RMSE with Ridge: {round(math.sqrt(np.mean(-MSEs)) ,4)}' )
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<import_modules>
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
lasso=Lasso()
lasso_reg=ms.GridSearchCV(lasso, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 lasso_reg.fit(x_train,y_train)
 print(f'The best value of Alpha is: {lasso_reg.best_params_}' )
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image<define_variables>
lasso_mod=Lasso(alpha=0.0009)
 lasso_mod.fit(x_train,y_train)
 y_lasso_train=lasso_mod.predict(x_train)
 y_lasso_test=lasso_mod.predict(x_test)
 print(f'Root Mean Square Error train {str(math.sqrt(sklm.mean_squared_error(y_train, y_lasso_train)))}')
 print(f'Root Mean Square Error test {score(y_lasso_test)}' )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )<define_variables>
print(f"Lasso kept {sum(coefs != 0)} important features and dropped the other {sum(coefs == 0)} features" )
seed_everything(42 )<set_options>
Lasso_CV=Lasso(alpha=0.0009)
 MSEs=ms.cross_val_score(Lasso_CV, x, y, scoring='neg_mean_squared_error', cv=5)
 print("RMSE scores of the 5 folds:")
 for i,j in enumerate(MSEs):
j= math.sqrt(np.mean(-j))
print(f'Fold {i}: {round(j,4)}')
 print(f'Mean RMSE with Lasso: {round(math.sqrt(np.mean(-MSEs)) ,4)}')
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<init_hyperparams>
from sklearn.linear_model import ElasticNetCV
Dropout_model = 0.3855
FVC_weight = 0.2
Confidence_weight = 0.2<create_dataframe>
alphas = [0.000542555]
l1ratio = [0.1, 0.3,0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print(f'Root Mean Square Error test = {score(ela_pred)}')
 print(elastic_cv.alpha_)
 print(elastic_cv.l1_ratio_ )
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'] )<feature_engineering>
def regularization(x,y,modelo=Ridge, scaler=RobustScaler):

 y=np.array(target)
 x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0)
 scaler= scaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test)
 if modelo != ElasticNetCV:
if modelo == Ridge:
parameters= {'alpha':[x for x in range(1,101)]}
elif modelo == Lasso:
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
model=modelo()
model=ms.GridSearchCV(model, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 model.fit(x_train,y_train)
 y_pred= model.predict(x_test)
 print("The best RMSE score achieved with %s is: %s " %(model.best_params_,
score(y_pred)))
 elif modelo == ElasticNetCV:
alphas = [0.000542555]
l1ratio = [0.1, 0.3,0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train)
 ela_pred=elasticmod.predict(x_test)
 print("The best RMSE score achieved with alpha %s and l1_ratio %s is: %s "
%(elastic_cv.alpha_,elastic_cv.l1_ratio_, score(ela_pred)))
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111<feature_engineering>
from xgboost.sklearn import XGBRegressor
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105<feature_engineering>
xgb= XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=3, min_child_weight=0, missing=None, n_estimators=4000,
n_jobs=1, nthread=None, objective='reg:squarederror', random_state=0,
reg_alpha=0.0001, reg_lambda=0.01, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 xgmod=xgb.fit(x_train,y_train)
 xg_pred=xgmod.predict(x_test)
 print(f'Root Mean Square Error test = {score(xg_pred)}' )
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158<feature_engineering>
cb_model = CatBoostRegressor(iterations=500,
learning_rate=0.05,
depth=10,
eval_metric='RMSE',
random_seed = 42,
bagging_temperature = 0.2,
od_type='Iter',
metric_period = 50,
od_wait=20 )
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107<feature_engineering>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 200,
"learning_rate" : 0.009,
"bagging_fraction" : 0.6,
"feature_fraction" : 0.2,
"bagging_frequency" : 4,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgtrain, lgval],
early_stopping_rounds=100,
verbose_eval=150,
evals_result=evals_result)
 lgb_pred=model.predict(x_test)
 pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return lgb_pred,pred_test_y, model, evals_result
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125<feature_engineering>
lgb_pred,pred_test, lgb_model, evals_result = run_lgb(x_train, y_train, x_test, y_test, X_test )
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089<feature_engineering>
print(f'Root Mean Square Error test = {score(lgb_pred)}' )
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100<feature_engineering>
vote_mod = VotingRegressor([('Ridge', ridge_mod),('Lasso', lasso_mod),('Elastic', elastic_cv), 
('XGBRegressor', xgb)])
 vote= vote_mod.fit(x_train, y_train.ravel())
 vote_pred=vote.predict(x_test)
 print(f'Root Mean Square Error test = {score(vote_pred)}' )
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096<feature_engineering>
stregr = StackingRegressor(regressors=[elastic_cv,ridge_mod, lasso_mod, vote_mod],
meta_regressor=xgb, use_features_in_secondary=True 
)
 stack_mod=stregr.fit(x_train, y_train.ravel())
 stacking_pred=stack_mod.predict(x_test)
 print(f'Root Mean Square Error test = {score(stacking_pred)}' )
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100<feature_engineering>
final_test=(0.25*vote_pred+0.35*stacking_pred+ 0.2*y_lasso_test + lgb_pred*0.2)
 print(f'Root Mean Square Error test= {score(final_test)}')
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
vote_test = vote_mod.predict(X_test)
 final1=np.expm1(vote_test)
 stack_test = stregr.predict(X_test)
 final2=np.expm1(stack_test)
 lasso_test = lasso_mod.predict(X_test)
 final3=np.expm1(lasso_test)
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
final=(0.2*final1+0.6*final2+0.2*final3)
 final_submission = pd.DataFrame({
"Id": b["Id"],
"SalePrice": final
})
 final_submission.to_csv("final_submission.csv", index=False)
 final_submission.head()
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283<feature_engineering>
sns.set_style('whitegrid')
 warnings.simplefilter(action='ignore')
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087<feature_engineering>
a = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 b = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090<feature_engineering>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
print('The shape of our training set: %s houses and %s features'%(a.shape[0],a.shape[1]))
print('The shape of our training set: %s houses and %s features'%(b.shape[0],b.shape[1]))
print('The testing set has 1 feature less than the training set, which is SalePrice, the target to predict ' )
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"Total size is {c1.shape}" )
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402<feature_engineering>
msv1(c1, 20, color=sns.color_palette('Reds',15))
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1<feature_engineering>
c=c1.dropna(thresh=len(c1)*0.8, axis=1)
 print(f"We dropped {c1.shape[1]-c.shape[1]} features in the combined set" )
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0<load_from_csv>
NA=c[allna.index.to_list() ]
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )<concatenate>
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )<feature_engineering>
ffill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
msv2(c )
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
d=filling_NA(c, ffill_cols )
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )<split>
fill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )<compute_test_metric>
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
for col in NAcols:
if c[col].dtype != "object":
c[col]= c[col].fillna(0 )
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
c['TotalArea'] = c['TotalBsmtSF'] + c['1stFlrSF'] + c['2ndFlrSF'] + c['GrLivArea'] +c['GarageArea']
c['Bathrooms'] = c['FullBath'] + c['HalfBath']*0.5
c['Year average']=(c['YearRemodAdd']+c['YearBuilt'])/2
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
c['MSSubClass'] = c['MSSubClass'].apply(str)
 c['YrSold'] = c['YrSold'].astype(str )
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
cb=pd.get_dummies(c)
 print(f"the shape of the original dataset {c.shape}")
 print(f"the shape of the encoded dataset {cb.shape}")
 print(f"We have {cb.shape[1]- c.shape[1]} new encoded features" )
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
Train = cb[:na]
Test = cb[na:]
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
a['GrLivArea'].sort_values(ascending=False ).head(2 )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
a['TotalBsmtSF'].sort_values(ascending=False ).head(1 )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
a['MasVnrArea'].sort_values(ascending=False ).head(1 )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
a['1stFlrSF'].sort_values(ascending=False ).head(1 )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
a['TotRmsAbvGrd'].sort_values(ascending=False ).head(1 )
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
train=Train[(Train['GrLivArea'] < 4600)&(Train['MasVnrArea'] < 1500)]
print(f'We removed {Train.shape[0]- train.shape[0]} outliers' )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
target=a[['SalePrice']]
target.loc[1298]
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
target.loc[523]
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
pos = [1298,523, 297]
target.drop(target.index[pos], inplace=True )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
print('We make sure that both train and target sets have the same row number after removing the outliers:')
 print('Train: ',train.shape[0], 'rows')
 print('Target:', target.shape[0],'rows' )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
from scipy.stats import skew
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
target["SalePrice"] = np.log1p(target["SalePrice"] )
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
x=train
y=target.SalePrice.values
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=42 )
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
scaler= RobustScaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
def score(y_pred):
return str(math.sqrt(sklm.mean_squared_error(y_test, y_pred)) )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
ridge=Ridge()
parameters= {'alpha':[x for x in range(1,101)]}
ridge_reg=ms.GridSearchCV(ridge, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 ridge_reg.fit(x_train,y_train)
 print(f"The best value of Alpha is: {ridge_reg.best_params_}")
 print(f"The best score achieved with Alpha=11 is: {math.sqrt(-ridge_reg.best_score_)}")
 ridge_pred=math.sqrt(-ridge_reg.best_score_ )
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
ridge_mod=Ridge(alpha=18)
 ridge_mod.fit(x_train,y_train)
 y_pred_train=ridge_mod.predict(x_train)
 y_pred_test=ridge_mod.predict(x_test)
 print(f'Root Mean Square Error train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_pred_train)))}')
 print(f'Root Mean Square Error test = {score(y_pred_test)}' )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
Ridge_CV=Ridge(alpha=18)
 MSEs=ms.cross_val_score(Ridge_CV, x, y, scoring='neg_mean_squared_error', cv=5)
 print("RMSE scores of the 5 folds:")
 for i,j in enumerate(MSEs):
j= math.sqrt(np.mean(-j))
print(f'Fold {i}: {round(j,4)}')
 print(f'Mean RMSE with Ridge: {round(math.sqrt(np.mean(-MSEs)) ,4)}' )
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()<save_to_csv>
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
lasso=Lasso()
lasso_reg=ms.GridSearchCV(lasso, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 lasso_reg.fit(x_train,y_train)
 print(f'The best value of Alpha is: {lasso_reg.best_params_}' )
df.to_csv('submission.csv', index=False )<install_modules>
lasso_mod=Lasso(alpha=0.0009)
 lasso_mod.fit(x_train,y_train)
 y_lasso_train=lasso_mod.predict(x_train)
 y_lasso_test=lasso_mod.predict(x_test)
 print(f'Root Mean Square Error train {str(math.sqrt(sklm.mean_squared_error(y_train, y_lasso_train)))}')
 print(f'Root Mean Square Error test {score(y_lasso_test)}' )
!pip install.. /input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<set_options>
print(f"Lasso kept {sum(coefs != 0)} important features and dropped the other {sum(coefs == 0)} features" )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(42 )<normalization>
Lasso_CV=Lasso(alpha=0.0009)
 MSEs=ms.cross_val_score(Lasso_CV, x, y, scoring='neg_mean_squared_error', cv=5)
 print("RMSE scores of the 5 folds:")
 for i,j in enumerate(MSEs):
j= math.sqrt(np.mean(-j))
print(f'Fold {i}: {round(j,4)}')
 print(f'Mean RMSE with Lasso: {round(math.sqrt(np.mean(-MSEs)) ,4)}')
def load_scan(path):
paths = glob.glob(path+'/*.dcm')
 paths.sort(key=lambda f: int(re.sub('\D', '', f)))
 if len(paths)< 51:
paths = paths[:50]
slices = [pydicom.read_file(p)for p in paths]
try:
slices.sort(key=lambda x: float(x.ImagePositionPatient[2]))
except:
pass
image = np.stack([s.pixel_array.astype(float)for s in slices])
 return image, slices[0], slices<sort_values>
from sklearn.linear_model import ElasticNetCV
IN_TRAIN = '.. /input/osic-pulmonary-fibrosis-progression/test/'
paths = os.listdir(IN_TRAIN)
 paths.sort()<load_from_csv>
alphas = [0.000542555]
l1ratio = [0.1, 0.3,0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print(f'Root Mean Square Error test = {score(ela_pred)}')
 print(elastic_cv.alpha_)
 print(elastic_cv.l1_ratio_ )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient")
 sub['WHERE'] = 'test'
test['WHERE'] = 'sub'
train['WHERE'] = 'train'
data = train.append([test, sub] )<feature_engineering>
def regularization(x,y,modelo=Ridge, scaler=RobustScaler):

 y=np.array(target)
 x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0)
 scaler= scaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test)
 if modelo != ElasticNetCV:
if modelo == Ridge:
parameters= {'alpha':[x for x in range(1,101)]}
elif modelo == Lasso:
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
model=modelo()
model=ms.GridSearchCV(model, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 model.fit(x_train,y_train)
 y_pred= model.predict(x_test)
 print("The best RMSE score achieved with %s is: %s " %(model.best_params_,
score(y_pred)))
 elif modelo == ElasticNetCV:
alphas = [0.000542555]
l1ratio = [0.1, 0.3,0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train)
 ela_pred=elasticmod.predict(x_test)
 print("The best RMSE score achieved with alpha %s and l1_ratio %s is: %s "
%(elastic_cv.alpha_,elastic_cv.l1_ratio_, score(ela_pred)))
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 COLS = ['Sex','SmokingStatus']
FE = []
FE += ['Patient']
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 FE += ['age','percent','week','BASE', 'WHERE']
meta_df = data[FE]
meta_df['fold'] = 0
meta_df = meta_df[meta_df.WHERE=='test']
meta_df.head()<define_variables>
from xgboost.sklearn import XGBRegressor
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector)
 targets = []
tab = []
P = []
for i, p in enumerate(test.Patient.unique()):
sub = test.loc[test.Patient == p, :]
tab.append(get_tab(sub))
patients = pd.DataFrame({'Patient': test.Patient.unique() , 'target': 0, 'meta': tab, 'fold': 0})
 patients.head()<feature_engineering>
xgb= XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=3, min_child_weight=0, missing=None, n_estimators=4000,
n_jobs=1, nthread=None, objective='reg:squarederror', random_state=0,
reg_alpha=0.0001, reg_lambda=0.01, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 xgmod=xgb.fit(x_train,y_train)
 xg_pred=xgmod.predict(x_test)
 print(f'Root Mean Square Error test = {score(xg_pred)}' )
train['fold'] = 0<import_modules>
cb_model = CatBoostRegressor(iterations=500,
learning_rate=0.05,
depth=10,
eval_metric='RMSE',
random_seed = 42,
bagging_temperature = 0.2,
od_type='Iter',
metric_period = 50,
od_wait=20 )
class FCN(torch.nn.Module):
def __init__(self, base, in_f, num_classes, in_meta, dropout=True):
super(FCN, self ).__init__()
self.base = base
self.after_model = nn.Sequential(
 nn.Flatten() ,
nn.Dropout(0.5)
)
 self.meta = nn.Sequential(
 nn.Linear(in_meta, 100),
nn.ReLU() ,
nn.Linear(100, 100),
nn.ReLU() , 
)
 self.classification_meta = nn.Sequential(
 nn.Linear(in_f+100, num_classes)
)
 self.classification = nn.Sequential(
 nn.Linear(in_f, num_classes)
)
 self.meta_head = nn.Linear(100, num_classes)
 def forward(self, x, meta):
x = self.base(x)
 x = self.after_model(x)
 meta = self.meta(meta)
 features = torch.cat(( x,meta),dim=1)
 x = self.classification_meta(features)
 return x
def create_model(name, path=None):
model = get_model(name, pretrained=False)
 try:
features = list(model.children())[-1].in_features
except:
features = list(model.children())[-1][-1].in_features
model = nn.Sequential(*list(model.children())[:-1])
 model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))
model = FCN(model, features, config.n_seg_classes, config.meta_features, dropout=True)
 if path:
print('loading pretrained model {}'.format(path))
pretrain = torch.load(path)['model_state']
model.load_state_dict(pretrain)
 return model<import_modules>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 200,
"learning_rate" : 0.009,
"bagging_fraction" : 0.6,
"feature_fraction" : 0.2,
"bagging_frequency" : 4,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgtrain, lgval],
early_stopping_rounds=100,
verbose_eval=150,
evals_result=evals_result)
 lgb_pred=model.predict(x_test)
 pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return lgb_pred,pred_test_y, model, evals_result
class FCN2(torch.nn.Module):
def __init__(self, base, in_f, num_classes, in_meta, dropout=True):
super(FCN2, self ).__init__()
self.base = base
self.after_model = nn.Sequential(
 nn.Flatten() ,
nn.Dropout(0.5)
)
 self.meta = nn.Sequential(
 nn.Linear(in_meta, 1024),
nn.ReLU() ,
nn.Linear(1024, 512),
nn.ReLU() , 
)
 self.classification_meta = nn.Sequential(
 nn.Linear(in_f+512, num_classes)
)
 self.classification = nn.Sequential(
 nn.Linear(in_f, num_classes)
)
 self.meta_head = nn.Linear(100, num_classes)
 def forward(self, x, meta):
x = self.base(x)
 x = self.after_model(x)
 meta = self.meta(meta)
 features = torch.cat(( x,meta),dim=1)
 x = self.classification_meta(features)
 return x
def create_model2(name, path=None):
model = get_model(name, pretrained=False)
 try:
features = list(model.children())[-1].in_features
except:
features = list(model.children())[-1][-1].in_features
model = nn.Sequential(*list(model.children())[:-1])
 model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))
model = FCN2(model, features, config.n_seg_classes, config.meta_features, dropout=True)
 if path:
print('loading pretrained model {}'.format(path))
pretrain = torch.load(path)['model_state']
model.load_state_dict(pretrain)
 return model<find_best_params>
lgb_pred,pred_test, lgb_model, evals_result = run_lgb(x_train, y_train, x_test, y_test, X_test )
class config:
input_D = 30
input_H = 512
input_W = 512
n_seg_classes = 3
meta_features = 9
seed = 42
quantiles =(0.2, 0.5, 0.8)
 nets=[]
model_name = 'resnet50'
model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld1-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld2-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld3-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld4-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld5-2(2 ).pth')
 model.cuda()
nets.append(model)
 model_name = 'resnet50'
model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld1-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld2-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld3-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld4-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld5-2(4 ).pth')
 model.cuda()
nets.append(model)
 model_name = 'resnet50'
model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld1-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld2-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld3-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld4-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld5-2(5 ).pth')
 model.cuda()
nets.append(model )<concatenate>
print(f'Root Mean Square Error test = {score(lgb_pred)}' )
transform = A.Compose([
A.HorizontalFlip(p=1.0)
 ])
 transform2 = A.Compose([
A.VerticalFlip(p=1.0)
 ] )<categorify>
vote_mod = VotingRegressor([('Ridge', ridge_mod),('Lasso', lasso_mod),('Elastic', elastic_cv), 
('XGBRegressor', xgb)])
 vote= vote_mod.fit(x_train, y_train.ravel())
 vote_pred=vote.predict(x_test)
 print(f'Root Mean Square Error test = {score(vote_pred)}' )
def convertToHU(image, data):
image =(image + data.RescaleIntercept)/(data.RescaleSlope)
 return image
A_test, B_test, P_test, WEEK = {},{},{},{}
last_pat = ''
o1=[]
model.eval()
with torch.no_grad() :
t = tqdm(meta_df.Patient.values)
 for i,patient in enumerate(t):
meta_batch = torch.from_numpy(meta_df.values[i][1:-2][None].astype(np.float32)).cuda().float()
if patient != last_pat:
last_pat = patient
patient_dir = '.. /input/osic-pulmonary-fibrosis-progression/test/' + patient
imgs, meta, slices = load_scan(patient_dir)
 for i,img in enumerate(imgs):
imgs[i] = convertToHU(img, meta)
 imgs = crop(imgs)
 imgs = resize(imgs, shape=(50,512,512))
imgs = imgs[10:40]
imgs1,imgs2,imgs3 = np.zeros(( 30,512,512,1)) ,np.zeros(( 30,512,512,1)) ,np.zeros(( 30,512,512,1))
for i,img in enumerate(imgs):
imgs1[i] = window(img, -600, 1500)[:,:,None]
imgs2[i] = window(img, 100, 700)[:,:,None]
imgs3[i] = window(img, 40, 400)[:,:,None]
imgs_ = np.concatenate([imgs1, imgs2, imgs3], axis=-1)
 image_batch = []
for i in range(1):
imgs = imgs_[15]
imgs = np.rollaxis(imgs, -1, 0)/ 255.
 image_batch.append(imgs)
 break
image_batch = np.array(image_batch)
 p = p[0]
o=[]
for model in nets:
for img in image_batch:
img = img[None]
img = torch.from_numpy(img ).cuda().float()
o.append(model(img, meta_batch ).cpu().numpy())
 for o in np.mean(np.array(o), axis=0):
o1.append(o)
 o1=np.array(o1 )<prepare_output>
stregr = StackingRegressor(regressors=[elastic_cv,ridge_mod, lasso_mod, vote_mod],
meta_regressor=xgb, use_features_in_secondary=True 
)
 stack_mod=stregr.fit(x_train, y_train.ravel())
 stacking_pred=stack_mod.predict(x_test)
 print(f'Root Mean Square Error test = {score(stacking_pred)}' )
sub = data[data.WHERE=='test'].copy()
sub['FVC'] = o1[:, 1]
sub['Confidence'] = o1[:, 2] - o1[:, 0]
subm = sub[['Patient_Week','FVC','Confidence']].copy()
subm.head() 
<load_from_csv>
final_test=(0.25*vote_pred+0.35*stacking_pred+ 0.2*y_lasso_test + lgb_pred*0.2)
 print(f'Root Mean Square Error test= {score(final_test)}')
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<prepare_output>
vote_test = vote_mod.predict(X_test)
 final1=np.expm1(vote_test)
 stack_test = stregr.predict(X_test)
 final2=np.expm1(stack_test)
 lasso_test = lasso_mod.predict(X_test)
 final3=np.expm1(lasso_test)
sub = subm.copy()<save_to_csv>
final=(0.2*final1+0.6*final2+0.2*final3)
 final_submission = pd.DataFrame({
"Id": b["Id"],
"SalePrice": final
})
 final_submission.to_csv("final_submission.csv", index=False)
 final_submission.head()