Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression,RidgeCV, LassoCV, ElasticNetCV
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
submm = pd.read_csv('.. /input/osic36175/submission_6.175.csv')
 submm = submm.values
colnames = ['Patient','FVC_175', 'Confidence_175']
submm = pd.DataFrame(submm, columns=colnames )<install_modules>
train = pd.read_csv(".. /input/train.csv")
 print('The size of the train dataset is {}'.format(train.shape))
test = pd.read_csv(".. /input/test.csv")
 print('The size of the test dataset is {}'.format(test.shape))
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<set_options>
id_train = train['Id']
id_test = test['Id']
warnings.filterwarnings("ignore")
 %matplotlib inline
init_notebook_mode(connected=True)
 Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D,
Add, Conv2D, AveragePooling2D, LeakyReLU, Concatenate , Lambda 
)
 pd.set_option("display.max_columns", 100)
 custom_colors = ['
sns.palplot(sns.color_palette(custom_colors))<set_options>
train.drop('Id', axis = 1, inplace= True)
 test.drop('Id', axis = 1, inplace = True)
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<define_variables>
train.dtypes.value_counts()
EPOCHS = 30
BATCH_SIZE = 8
NFOLD = 5
LR = 0.003
SAVE_BEST = True
MODEL_CLASS = 'b1'
path = '.. /input/osic-pulmonary-fibrosis-progression/'<load_from_csv>
test.dtypes.value_counts()
tr = pd.read_csv('.. /input/osic-pf-train-meta-image/train_df_with_img_feat.csv')
 ts = pd.read_csv(f"{path}/test.csv")
 ts['Where'] = 'test'
ts_patient_ids = ts.Patient.unique().tolist()<define_variables>
train_rowsize = train.shape[0]
test_rowsize = test.shape[0]
test_rowsize
percentile_range_1 = np.linspace(0.3,0.6,11)
 percentile_range_2 = range(30,63, 3)
 percentile_range = [(round(x,2),y)for x,y in zip(percentile_range_1,percentile_range_2)]
percentile_range<prepare_output>
warnings.filterwarnings('ignore')
 data_all = pd.concat(( train, test))
data_all.drop('SalePrice', axis = 1, inplace = True)
image_data_df = pd.DataFrame(image_data)
 image_data_df.head()<merge>
null_values = data_all.columns[data_all.isnull().any() ]
null_features = data_all[null_values].isnull().sum().sort_values(ascending = False)
 missing_data = pd.DataFrame({'No of Nulls' :null_features})
 missing_data
ts = ts.merge(image_data_df, left_on = ['Patient'], right_on = ['Patient'], how = 'left')
 df = pd.concat([tr,ts] )<feature_engineering>
corrs['Abs_correlation'] = corrs.Correlations.abs()
corrs
variables = ['ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60','Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in variables:
median = df[var].quantile(q = 0.5, interpolation='linear')
 df.loc[df[var]==0,var] = median
median = df['Avg_TissueByLung_30_60'].quantile(q = 0.5, interpolation='linear')
 df.loc[df['Avg_TissueByLung_30_60']>=0.3,'Avg_TissueByLung_30_60'] = median
median = df['Avg_Tissue_thickness_30_60'].quantile(q = 0.5, interpolation='linear')
 df.loc[df['Avg_Tissue_thickness_30_60']>=20000,'Avg_Tissue_thickness_30_60'] = median
df["Avg_Tissue_30_60_Quartile"] = pd.qcut(df.Avg_Tissue_30_60, q = 4, labels = ['Q1','Q2','Q3','Q4'] )<feature_engineering>
corrs.sort_values('Correlations', ascending = False ).query('Abs_correlation>0.45' )
l = ['SliceThickness','PixelSpacing','ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60',
'Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in l:
df[var] =(df[var] - df[var].min())/(df[var].max() - df[var].min() )<count_unique_values>
missing_data = ['PoolQC',"MiscFeature","Alley", "Fence", "FireplaceQu", 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath',
'BsmtHalfBath', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', "MasVnrType", "MasVnrArea",
'MSSubClass']
for x in missing_data:
data_all[x] = data_all[x].fillna(0 )
quartile = ['Q1','Q2','Q3','Q4']
for q in quartile:
temp_df = train.loc[train.Avg_Tissue_30_60_Quartile==q,:]
patients = temp_df.Patient.unique()
fvc_diff_lst = []
print('Quartile: ', q)
 for patient in patients:
fvc_lst = temp_df.loc[temp_df.Patient == patient, 'FVC'].tolist()
fvc_diff = fvc_lst[-1]-fvc_lst[0]
fvc_diff_lst.append(fvc_diff)
 print('FVC Diff Mean: ', sum(fvc_diff_lst)/len(fvc_diff_lst))
print('FVC Diff Q1: ', np.quantile(fvc_diff_lst,.25))
print('FVC Diff Q2/Median: ', np.quantile(fvc_diff_lst,.5))
print('FVC Diff Q3: ', np.quantile(fvc_diff_lst,.75))
print('FVC Diff Q4: ', np.quantile(fvc_diff_lst,.90))<feature_engineering>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
data_all["LotFrontage"] = data_all.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img_1(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
data_all['MSZoning'].value_counts(normalize = True)
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class = None, fold=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(6,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/lindecay-efficientnetb1-models')if str(fold)in w][0]
model.load_weights('.. /input/lindecay-efficientnetb1-models/' + weights)
 return model<define_variables>
data_all['MSZoning'] = data_all['MSZoning'].fillna('RL')
 data_all['MSZoning'].isnull().any()
bst_quantile = [0.8, 0.5, 0.1, 0.1, 0.1]<load_from_csv>
data_all['Utilities'].value_counts(normalize = True )
model_class = 'b1'
NFOLD = 5
kf = KFold(n_splits=NFOLD, random_state=42,shuffle=False)
 P = np.array(P)
 subs = []
folds_history = []
for fold,(tr_idx, val_idx)in enumerate(kf.split(P)) :
print('
print('
print('
print('Predicting...')
 model = build_model(shape=(512, 512, 1), model_class = model_class, fold = fold)
 q = bst_quantile[fold]
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {}, {}, {}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 counter = 0
for i in ldir:
if int(i[:-4])/ len(ldir)< 0.7 and int(i[:-4])/ len(ldir)> 0.20:
x.append(get_img_1(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
counter+=1
if counter == 20:
break
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub)
 K.clear_session()
del model
gc.collect()<prepare_output>
data_all['Utilities'] = data_all['Utilities'].fillna('AllPub' )
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[i]["FVC"] *(1/N)
 sub["Confidence"] += subs[i]["Confidence"] *(1/N )<save_to_csv>
data_all['Functional'].value_counts(normalize = True )
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
data_all['Functional'] = data_all['Functional'].fillna('Typ' )
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<feature_engineering>
data_all['Electrical'] = data_all['Electrical'].fillna(data_all['Electrical'].mode() [0])
 data_all['KitchenQual'] = data_all['KitchenQual'].fillna(data_all['KitchenQual'].mode() [0])
 data_all['Exterior1st'] = data_all['Exterior1st'].fillna(data_all['Exterior1st'].mode() [0])
 data_all['Exterior2nd'] = data_all['Exterior2nd'].fillna(data_all['Exterior2nd'].mode() [0])
 data_all['SaleType'] = data_all['SaleType'].fillna(data_all['SaleType'].mode() [0] )
tr = pd.read_csv(f"{path}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{path}/test.csv")
 tr_patient_ids = tr.Patient.unique().tolist()
ts_patient_ids = chunk.Patient.unique().tolist()
sub = pd.read_csv(f"{path}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
null_values_2 = data_all.columns[data_all.isnull().any() ]
null_features_2 = data_all[null_values_2].isnull().sum().sort_values(ascending = False)
 missing_data_2 = pd.DataFrame({'No of Nulls' :null_features_2})
 missing_data_2
print('|\t\t NO MORE MISSING VALUES REMAINING.
 \t\t\t...IMPUTING COMPLETED...' )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
meta_data = tr.append([chunk, sub] )<count_unique_values>
data_all.dtypes.value_counts()
print(tr.shape, chunk.shape, sub.shape, meta_data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
meta_data.Patient.nunique() )<feature_engineering>
train_new.dtypes.value_counts()
meta_data['min_week'] = meta_data['Weeks']
meta_data.loc[meta_data.WHERE=='test','min_week'] = np.nan<feature_engineering>
test_new.dtypes.value_counts()
meta_data['min_week'] = meta_data['Weeks']
meta_data.loc[meta_data.WHERE=='test','min_week'] = np.nan
meta_data['min_week'] = meta_data.groupby('Patient')['min_week'].transform('min' )<merge>
train_dummies = pd.get_dummies(train_new, columns = train_categcols)
 test_dummies = pd.get_dummies(test_new, columns = test_categcols)
 train_encoded, test_encoded = train_dummies.align(test_dummies, join = 'left', axis = 1)
 print('\t\tShape of the new encoded train: {}'.format(train_encoded.shape))
print('
\t\tShape of the new encoded test: {}'.format(test_encoded.shape))
print('
\t\t\t.... Encoding completed.....' )
base = meta_data.loc[meta_data.Weeks == meta_data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 meta_data = meta_data.merge(base, on='Patient', how='left')
 base = meta_data.loc[meta_data.Weeks == meta_data.min_week]
base = base[['Patient','Percent']].copy()
base.columns = ['Patient','min_Percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 meta_data = meta_data.merge(base, on='Patient', how='left')
 meta_data['base_week'] = meta_data['Weeks'] - meta_data['min_week']
del base<data_type_conversions>
train_encoded.dtypes.value_counts()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in meta_data[col].unique() :
print(col, mod)
 FE.append(mod)
 meta_data[mod] =(meta_data[col] == mod ).astype(int)
<prepare_output>
for col in skew_cols.index.tolist() :
train_encoded[col] = np.log1p(train_encoded[col])
 test_encoded[col] = test_encoded[col].apply(np.log1p)
 print(test_encoded.dtypes.value_counts())
 print('
\t\t:)Skewed data Transformation Completed :)' )
image_data_df = pd.DataFrame(image_data)
 image_data_df.head()<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(train_encoded, predictor,
test_size=0.3, random_state=42)
 print(X_train.shape)
 print(y_train.shape)
 print(X_test.shape)
 print(y_test.shape )
tr1 = pd.read_csv('.. /input/osic-pf-train-meta-image/train_df_with_img_feat.csv')
 cols = image_data_df.columns
tr1 = tr1[cols]
tr1 = tr1.drop_duplicates(keep = 'first')
 tr1.head()<concatenate>
def rmse(true_data, predicted_data):
return np.sqrt(mean_squared_error(true_data, predicted_data))
image_data_df = pd.concat([tr1,image_data_df])
 image_data_df.head()<feature_engineering>
linearRegression = LinearRegression().fit(X_train, y_train)
 prediction1 = linearRegression.predict(X_test)
 LR_score = linearRegression.score(X_test, y_test)
 LR_rmse = rmse(y_test, prediction1)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\t\tThe score is: ',LR_score*100)
 print('\t\tThe rmse is : ',LR_rmse*100 )
variables = ['ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60','Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in variables:
median = image_data_df[var].quantile(q = 0.5, interpolation='linear')
 image_data_df.loc[image_data_df[var]==0,var] = median
median = image_data_df['Avg_TissueByLung_30_60'].quantile(q = 0.5, interpolation='linear')
 image_data_df.loc[image_data_df['Avg_TissueByLung_30_60']>=0.3,'Avg_TissueByLung_30_60'] = median
median = image_data_df['Avg_Tissue_thickness_30_60'].quantile(q = 0.5, interpolation='linear')
 image_data_df.loc[image_data_df['Avg_Tissue_thickness_30_60']>=20000,'Avg_Tissue_thickness_30_60'] = median<merge>
alphas = [0.005, 0.05, 0.1, 1, 5, 10, 50, 100]
final_data = meta_data.merge(image_data_df, left_on = ['Patient','WHERE'], right_on = ['Patient', 'Data'], how = 'left')
 print(final_data.shape)
 print(final_data.shape)
 final_data.head()<feature_engineering>
ridgeCV = RidgeCV(alphas=alphas ).fit(X_train, y_train)
 prediction2 = ridgeCV.predict(X_test)
 R_score = ridgeCV.score(X_test, y_test)
 R_rmse = rmse(y_test, prediction2)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here was alpha = {}
'.format(ridgeCV.alpha_))
print('\t\tThe score is: ',R_score*100)
 print('\t\tThe rmse is : ',R_rmse*100 )
final_data['age'] =(final_data['Age'] - final_data['Age'].min())/(final_data['Age'].max() - final_data['Age'].min())
 final_data['BASE_FVC'] =(final_data['min_FVC'] - final_data['min_FVC'].min())/(final_data['min_FVC'].max() - final_data['min_FVC'].min())
 final_data['week'] =(final_data['base_week'] - final_data['base_week'].min())/(final_data['base_week'].max() - final_data['base_week'].min())
 final_data['BASE_percent'] =(final_data['min_Percent'] - final_data['min_Percent'].min())/(final_data['min_Percent'].max() - final_data['min_Percent'].min())
 l = ['SliceThickness','PixelSpacing','ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60',
'Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in l:
final_data[var] =(final_data[var] - final_data[var].min())/(final_data[var].max() - final_data[var].min())
 FE += ['age','week','BASE_FVC','BASE_percent']
FE.extend(l )<split>
lassoCV = LassoCV(alphas=[0.005, 0.001, 0.05, 0.01,1, 5], max_iter=1e2 ).fit(X_train, y_train)
 prediction3 = lassoCV.predict(X_test)
 L_score = lassoCV.score(X_test, y_test)
 L_rmse = rmse(y_test, prediction3)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here was alpha = {}'.format(lassoCV.alpha_))
print('
\t\tThe score is: ',L_score*100)
 print('\t\tThe rmse is : ',L_rmse*100 )
tr = final_data.loc[final_data.WHERE=='train']
chunk = final_data.loc[final_data.WHERE=='val']
sub = final_data.loc[final_data.WHERE=='test']<compute_test_metric>
l1_ratios = np.linspace(0.1, 0.9, 9)
 elasticnetCV = ElasticNetCV(alphas=alphas, l1_ratio=l1_ratios, max_iter=1e2 ).fit(X_train, y_train)
 prediction4 = elasticnetCV.predict(X_test)
 EN_score = elasticnetCV.score(X_test, y_test)
 EN_rmse = rmse(y_test, prediction4)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here was alpha = {} and l1_ratios = {}
'.format(elasticnetCV.alpha_, elasticnetCV.l1_ratio_))
print('\t\tThe score is: ',EN_score*100)
 print('\t\tThe rmse is : ',EN_rmse*100 )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):

 sd_clipped = np.maximum(confidence, 70)
 delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
 metric = - np.sqrt(2)* delta / sd_clipped - np.log(np.sqrt(2)* sd_clipped)
 if return_values:
return metric
else:
return np.mean(metric)
 def make_model(FE):
z = Input(( len(FE),), name="Patient")
 x = Dense(100, activation="relu", name="d1" )(z)
 x = Dense(100, activation="relu", name="d2" )(x)
 p1 = Dense(3, activation="linear", name="p1" )(x)
 p2 = Dense(3, activation="relu", name="p2" )(x)
 preds = Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
randfr = RandomForestRegressor(random_state = 42 )
y = tr['FVC'].values.astype(float)
 X_train = tr[FE].values
X_test = sub[FE].values
train_preds = np.zeros(( X_train.shape[0], 3))
test_preds = np.zeros(( X_test.shape[0], 3))<define_variables>
param_grid = {
'n_estimators': [50,250,500],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [2, 4, 6, 8, 10],
}
%%time
FE = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes', 'age', 'week', 'BASE_FVC', 'BASE_percent']
image_features = ['SliceThickness','PixelSpacing','ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60',
'Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
FE1 = FE
FE2 = FE+['ApproxVol_30_60']
FE3 = FE+['Avg_Tissue_thickness_30_60']
FE4 = FE+['Avg_TissueByLung_30_60']
FE5 = FE+['ApproxVol_30_60','Avg_Tissue_thickness_30_60','Avg_TissueByLung_30_60']
model_cnt = 1
for FE in [FE1, FE2, FE3, FE4, FE5]:
print('Model Count: ', model_cnt, '
Features Used: ', FE)
 y = tr['FVC'].values.astype(float)
 X_train = tr[FE].values
X_test = sub[FE].values
globals() ['train_preds_{}'.format(model_cnt)] = np.zeros(( X_train.shape[0], 3))
globals() ['test_preds_{}'.format(model_cnt)] = np.zeros(( X_test.shape[0], 3))
NFOLD = 5
gkf = GroupKFold(n_splits=NFOLD)
 groups = tr['Patient'].values
cnt = 0
EPOCHS = 800
BATCH_SIZE=128
for tr_idx, val_idx in gkf.split(X_train, y, groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(FE)
 net.fit(X_train[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(X_train[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(X_train[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(X_train[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 globals() ['train_preds_{}'.format(model_cnt)][val_idx] = net.predict(X_train[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 globals() ['test_preds_{}'.format(model_cnt)] += net.predict(X_test, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
predicted_fvc = globals() ['train_preds_{}'.format(model_cnt)][:,1]
confidence = globals() ['train_preds_{}'.format(model_cnt)][:,2]-globals() ['train_preds_{}'.format(model_cnt)][:,0]
model_score = laplace_log_likelihood(actual_fvc = y, predicted_fvc = predicted_fvc, confidence = confidence,
return_values = False)
 print('Overall Score: ', model_score)
 model_cnt+=1 
<define_search_space>
randfr_cv = GridSearchCV(estimator=randfr, param_grid=param_grid, cv= 5)
 randfr = randfr_cv.fit(X_train, y_train )
scores = []
for i in range(1,6):
globals() ['predicted_fvc_{}'.format(i)] = globals() ['train_preds_{}'.format(i)][:,1]
globals() ['confidence_{}'.format(i)] = globals() ['train_preds_{}'.format(i)][:,2]-globals() ['train_preds_{}'.format(i)][:,0]
for i1 in np.linspace(0,0.5,11):
for i2 in np.linspace(0,0.5,11):
for i3 in np.linspace(0,0.5,11):
for i4 in np.linspace(0,0.5,11):
for i5 in np.linspace(0,0.5,11):
if i1+i2+i3+i4+i5 == 1:
train_preds =(i1*train_preds_1+
i2*train_preds_2+
i3*train_preds_3+
i4*train_preds_4+
i5*train_preds_5)
 predicted_fvc = train_preds[:,1]
confidence = train_preds[:,2]-train_preds[:,0]
score = laplace_log_likelihood(actual_fvc = y, predicted_fvc = predicted_fvc, confidence = confidence, return_values = False)
 scores.append(( i1,i2,i3,i4,i5,score))
scores = sorted(scores, key = lambda x: x[-1])
 scores[-10:]<compute_test_metric>
prediction5 = randfr.predict(X_test)
 RF_score = randfr.score(X_test, y_test)
 RF_rmse = rmse(y_test, prediction5)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here were = {}
'.format(randfr_cv.best_params_))
print('\t\tThe score for random forest is: {} '.format(RF_score*100))
print('\t\tThe rmse is: {} '.format(RF_rmse*100))
best_weights = scores[-1]
train_preds =(best_weights[0]*train_preds_1+best_weights[1]*train_preds_2+best_weights[2]*train_preds_3+
best_weights[3]*train_preds_4+best_weights[4]*train_preds_5)
 test_preds =(best_weights[0]*test_preds_1+best_weights[1]*test_preds_2+best_weights[2]*test_preds_3+
best_weights[3]*test_preds_4+best_weights[4]*test_preds_5 )<compute_test_metric>
score_vals = [LR_score, R_score, L_score, EN_score, RF_score]
rmse_vals = [LR_rmse, R_rmse, L_rmse, EN_rmse, RF_rmse]
labels = ['Linear', 'Ridge', 'Lasso', 'ElasticNet', 'RandomForest']
rmse_df = pd.Series(score_vals, index=labels ).to_frame()
rmse_df.rename(columns={0: 'SCORES'}, inplace=1)
 rmse_df['RMSE'] = rmse_vals
rmse_df
sigma_opt = mean_absolute_error(y, train_preds[:,1])
 sigma_uncertain = train_preds[:,2] - train_preds[:,0]
sigma_mean = np.mean(sigma_uncertain)
 print(sigma_opt, sigma_mean)
<prepare_output>
rmse_df = rmse_df.sort_values(['RMSE'], ascending=True)
 rmse_df
sub['FVC1'] = test_preds[:, 1]
sub['Confidence1'] = test_preds[:,2] - test_preds[:,0]
submission = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
submission.loc[~submission.FVC1.isnull() ].head(10 )<feature_engineering>
start_xgb = datetime.now()
xgb = XGBRegressor().fit(X_train, y_train)
 end_xgb = datetime.now()
xgb_time = end_xgb - start_xgb
print('Duration for XGBoost: {}'.format(xgb_time))
submission.loc[~submission.FVC1.isnull() ,'FVC'] = submission.loc[~submission.FVC1.isnull() ,'FVC1']
if sigma_mean < 70:
submission['Confidence'] = sigma_opt
else:
submission.loc[~submission.FVC1.isnull() ,'Confidence'] = submission.loc[~submission.FVC1.isnull() ,'Confidence1']<feature_engineering>
prediction6 = xgb.predict(X_test)
 xgb_score = xgb.score(X_test, y_test)
 xgb_rmse = rmse(y_test, prediction6)
 print('The scoring and root mean squared error for XGBoost in percentage
')
 print('\t\tThe score is: ',xgb_score*100)
 print('\t\tThe rmse is : ',xgb_rmse*100 )
org_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(org_test)) :
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'FVC'] = org_test.FVC[i]
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'Confidence'] = 70<save_to_csv>
start_lgbm = datetime.now()
lgb = LGBMRegressor().fit(X_train, y_train)
 end_lgbm = datetime.now()
lgbm_time = end_lgbm - start_lgbm
print('Duration for Light GBM: {}'.format(lgbm_time))
submission[["Patient_Week","FVC","Confidence"]].to_csv("submission_meta.csv", index = False )<prepare_output>
prediction7 = lgb.predict(X_test)
 lgb_score = lgb.score(X_test, y_test)
 lgb_rmse = rmse(y_test, prediction7)
 print('The scoring and root mean squared error for light GBM in percentage
')
 print('\t\tThe score is: ',lgb_score*100)
 print('\t\tThe rmse is : ',lgb_rmse*100 )
reg_sub = submission[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
comparisons = {'Scores':(lgb_score, xgb_score), 'RMSE':(lgb_rmse, xgb_rmse), 'Execution Time' :(lgbm_time, xgb_time)}
comparisons_df = pd.DataFrame(comparisons )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
comparisons_df.index= ['LightGBM','XGBOOST']
comparisons_df
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.4*df1['FVC'] + 0.6*df2['FVC']
df['Confidence'] = 0.4*df1['Confidence'] + 0.6*df2['Confidence']
df.head()<save_to_csv>
test_encoded.isnull().sum()
df.to_csv('submission.csv', index=False )<import_modules>
null_values = test_encoded.columns[test_encoded.isnull().any() ]
null_features = test_encoded[null_values].isnull().sum().sort_values(ascending = False)
 missing = pd.DataFrame({'No of Nulls' :null_features})
 missing
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, GroupKFold<import_modules>
test_encoded = test_encoded.fillna(0 )
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<set_options>
prediction = lassoCV.predict(test_encoded)
 final_prediction = np.exp(prediction )
<define_variables><EOS> 
House_submission = pd.DataFrame({'Id': id_test, 'SalePrice': final_prediction})
 print(House_submission.shape)
 House_submission.to_csv('House_prediction.csv', index = False)