Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
%matplotlib inline
print("Setup Complete")
 warnings.filterwarnings('ignore' )
print(colored("Gender wise distribution of patients :","blue"))
print(patient_df['Sex'].value_counts() )<count_values>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',na_values='nan')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',na_values='nan')
 print("Train size:",df_train.shape)
 print("Test size:",df_test.shape)
 train_ID =df_train['Id']
test_ID =df_test['Id']
df_train.drop("Id", axis = 1, inplace = True)
 df_test.drop("Id", axis = 1, inplace = True)
 print("Size of train data after dropping Id: {}".format(df_train.shape))
print("Size of test data after dropping Id: {}".format(df_test.shape))
print(colored('Total Smoking counts', 'red'))
print(patient_df['SmokingStatus'].value_counts())
 print("
")
 print(colored("Male Smoking counts",'blue'))
print(patient_df[patient_df['Sex']=='Male']['SmokingStatus'].value_counts())
 print("
")
 print(colored("Female Smoking counts",'green'))
print(patient_df[patient_df['Sex']=='Female']['SmokingStatus'].value_counts() )<sort_values>
train = df_train.drop(df_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<300000)].index )
train_df[train_df['FVC'] > 5000].sort_values(by='FVC', ascending=False )<define_variables>
p_df=[]
for col in df_train_num.columns:
stat, p = shapiro(df_train[col])
 p_df.append(p)
 p_df_=pd.DataFrame(p_df,df_train_num.columns ).sort_values(by=0,ascending=True)
 p_df_=p_df_.loc[p_df_[0]>0.05]
p_df_
file_len = folder_len = 0
files = []
for dirpath, dirnames, filenames in os.walk(train_dir):
file_len += len(filenames)
 folder_len += len(dirnames)
 files.append(len(filenames))
print("Training folder contains", f'{file_len:,}', "CT scans for all patients.")
 print('Training folder have only',f'{folder_len:,}', "unique patients.")
 print("
")
 print('Each patient have', f'{round(np.mean(files)) :,}', 'average number of CT scans.')
 print('Maximum images per patient', f'{round(np.max(files)) :,}')
 print('Minimum images per patient', f'{round(np.min(files)) :,}' )<normalization>
ntrain = train.shape[0]
ntest = df_test.shape[0]
all_data = pd.concat(( train,df_test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
def load_scan(path):
slices = [pydicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 for s in slices:
s.SliceThickness = slice_thickness
return slices<data_type_conversions>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na} )
def get_pixels_hu(slices):
image = np.stack([s.pixel_array for s in slices])
 image = image.astype(np.int16)
 image[image == -2000] = 0
for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope * image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 return np.array(image, dtype=np.int16 )<features_selection>
all_data_rep=all_data.copy()
all_data_rep["PoolQC"] = all_data_rep["PoolQC"].fillna(0)
 all_data_rep["MiscFeature"] = all_data_rep["MiscFeature"].fillna(0)
 all_data_rep["Fence"] = all_data_rep["Fence"].fillna(0)
 all_data_rep["FireplaceQu"] = all_data_rep["FireplaceQu"].fillna(0)
 all_data_rep["LotFrontage"] = all_data_rep.groupby('Neighborhood')["LotFrontage"].transform(
 lambda x:x.fillna(x.median()))
grd_col=["GarageQual","GarageCond","GarageType","GarageFinish","GarageArea","GarageCars","GarageYrBlt"]
for col in grd_col:
all_data_rep[col] = all_data_rep[col].fillna(0)
 grd_col=["BsmtExposure","BsmtCond","BsmtQual","BsmtFinType1","BsmtFinType2","BsmtUnfSF","BsmtFinSF1","BsmtFinSF2"]
for col in grd_col:
all_data_rep[col] = all_data_rep[col].fillna(0)
 all_data_rep["TotalBsmtSF"] = all_data_rep["TotalBsmtSF"].fillna(0)
 all_data_rep["MasVnrArea"] = all_data_rep["MasVnrArea"].fillna(0)
 all_data_rep["MasVnrType"] = all_data_rep["MasVnrType"].fillna(0)
 all_data_rep['MSZoning']=all_data_rep['MSZoning'].fillna(all_data_rep['MSZoning'].mode() [0])
 all_data_rep["BsmtFullBath"] = all_data_rep["BsmtFullBath"].fillna(0)
 all_data_rep["BsmtHalfBath"] = all_data_rep["BsmtHalfBath"].fillna(0)
 all_data_rep["Functional"] =all_data_rep["Functional"].fillna("Typ")
 all_data_rep["Alley"] = all_data_rep["Alley"].fillna(0)
 mode_col = ['Electrical','KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType']
for col in mode_col:
all_data_rep[col] = all_data_rep[col].fillna(all_data_rep[col].mode() [0])
 all_data_rep["MoSold"] = all_data_rep["MoSold"].fillna(0 )
first_patient_scan = load_scan(train_dir + patient_ids[0] )<data_type_conversions>
all_data_rep.isnull().sum().sort_values(ascending=False)[:3]
def set_lungwin(img, hu=[-1200., 600.]):
lungwin = np.array(hu)
 newimg =(img-lungwin[0])/(lungwin[1]-lungwin[0])
 newimg[newimg < 0] = 0
newimg[newimg > 1] = 1
newimg =(newimg * 255 ).astype('uint8')
 return newimg<feature_engineering>
all_data_rep['TotalSF'] = all_data_rep['TotalBsmtSF'] + all_data_rep['1stFlrSF'] + all_data_rep['2ndFlrSF']
first_patient_scan_array = set_lungwin(get_pixels_hu(first_patient_scan))<set_options>
all_data_rep_enc_1=all_data_rep.copy()
imageio.mimsave("/tmp/gif.gif", first_patient_scan_array, duration=0.00001)
 Image(filename="/tmp/gif.gif", format='png' )<categorify>
mean_sale_subclass=train.groupby(['MSSubClass'] ).mean()
df_mean_sale_subclass=pd.DataFrame(mean_sale_subclass['SalePrice'].sort_values(ascending=True)).reset_index()
df_mean_sale_subclass[:2]
def segment_lung_mask(image):
segmented = np.zeros(image.shape)
 for n in range(image.shape[0]):
binary_image = np.array(image[n] > -320, dtype=np.int8)+1
labels = measure.label(binary_image)
 background_label_1 = labels[0,0]
background_label_2 = labels[0,-1]
background_label_3 = labels[-1,0]
background_label_4 = labels[-1,-1]
binary_image[background_label_1 == labels] = 2
binary_image[background_label_2 == labels] = 2
binary_image[background_label_3 == labels] = 2
binary_image[background_label_4 == labels] = 2
selem = disk(4)
 binary_image = closing(binary_image, selem)
 binary_image -= 1
binary_image = 1-binary_image
segmented[n] = binary_image.copy() * image[n]
return segmented<feature_engineering>
subclass_order=list(df_mean_sale_subclass['MSSubClass'])
 subclass_order
segmented_lungs = segment_lung_mask(first_patient_pixels )<drop_column>
qual_map = {0: 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
qual_col=['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond','ExterQual',
'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual']
for col in qual_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(qual_map)
 furnit_map = {0: 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ':6 }
furn_col=['BsmtFinType1', 'BsmtFinType2']
for col in furn_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(furnit_map)
 functional_map={0: 0, 'Sal': 1, 'Sev': 2, 'Maj2': 3, 'Maj1': 4, 'Mod': 5, 'Min2':6,'Min1': 7, 'Typ':8}
func_col=['Functional']
for col in func_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(functional_map)
 fence_map={0: 0, 'MnWw': 1, 'GdWo': 2, 'GdWo': 3, 'MnPrv': 4, 'GdPrv': 5}
fence_col=['Fence']
for col in fence_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(fence_map)
 bsm_exp_map={0: 0, 'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4}
all_data_rep_enc_1['BsmtExposure'] = all_data_rep_enc_1['BsmtExposure'].map(bsm_exp_map)
 garag_fin_map={0: 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}
all_data_rep_enc_1['GarageFinish'] = all_data_rep_enc_1['GarageFinish'].map(garag_fin_map)
 slope_map={'Sev': 1, 'Mod': 2, 'Gtl': 3}
all_data_rep_enc_1['LandSlope'] = all_data_rep_enc_1['LandSlope'].map(slope_map)
 shape_map={'IR3': 1, 'IR2': 2, 'IR1': 3, 'Reg': 4}
all_data_rep_enc_1['LotShape'] = all_data_rep_enc_1['LotShape'].map(shape_map)
 pave_map={'N': 1, 'P': 2, 'Y': 3}
all_data_rep_enc_1['PavedDrive'] = all_data_rep_enc_1['PavedDrive'].map(pave_map)
 street_map={0:0,'Pave': 1, 'Grvl': 2}
all_data_rep_enc_1['Street'] = all_data_rep_enc_1['Street'].map(street_map)
 alley_map={0: 0, 'Pave': 1, 'Grvl': 2}
all_data_rep_enc_1['Alley'] = all_data_rep_enc_1['Alley'].map(alley_map)
 air_map={ 'N': 1, 'Y': 2}
all_data_rep_enc_1['CentralAir'] = all_data_rep_enc_1['CentralAir'].map(air_map)
 class_map={0:0,30:1, 180:2, 45:3, 190:4, 90:5, 160:6, 50:7, 85:8, 40:9, 70:10, 80:11, 20:12, 75:13, 120:14, 150:14, 60:15}
all_data_rep_enc_1['MSSubClass'] = all_data_rep_enc_1['MSSubClass'].map(class_map)
 all_data_rep_enc_1['YrSold'].astype('int')
 mo_map={0:0, 4:1, 5:2, 6:7, 7:7.5, 8:9, 9:11, 10:2.5, 11:8.5, 2:8.5, 12:9.5, 1:1.5, 3:7.5}
all_data_rep_enc_1['MoSold'] = all_data_rep_enc_1['MoSold'].map(mo_map )
train_df.reset_index(inplace = True , drop = True)
 patient_df.reset_index(inplace = True , drop = True)
 test_df.reset_index(inplace = True , drop = True )<load_from_csv>
all_data_rep_enc_1.isnull().sum().sort_values(ascending=False)[:3]
<set_options>
train["SalePrice"] = np.log1p(train["SalePrice"])
 check_skewness('SalePrice' )
Dropout_model = 0.38559
FVC_weight = 0.2
Confidence_weight = 0.15
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<choose_model_class>
y_train=train["SalePrice"]
def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]<define_search_model>
all_data_rep_enc_1.dtypes[all_data_rep_enc_1.dtypes=='object'].index
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(5,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 return model<choose_model_class>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.3
for feat in skewed_features:
all_data_rep_enc_1[feat] = boxcox1p(all_data_rep_enc_1[feat], lam )
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )<define_variables>
all_data_rep_enc_1.dtypes[all_data_rep_enc_1.dtypes== "object"].index
BATCH_SIZE=128<merge>
col_ob=list(all_data_rep_enc_1.dtypes[all_data_rep_enc_1.dtypes== "object"].index)
 un=[]
for col in col_ob:
n=all_data_rep_enc_1[col].nunique()
un.append(n )
submission_df['Patient'] = submission_df['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission_df['Weeks'] = submission_df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission_df = submission_df[['Patient','Weeks','Confidence','Patient_Week']]
submission_df = submission_df.merge(test_df.drop('Weeks', axis=1), on="Patient")
 submission_df.head()<count_unique_values>
all_data_rep_enc_1_=all_data_rep_enc_1.copy()
train_df['WHERE'] = 'train'
test_df['WHERE'] = 'val'
submission_df['WHERE'] = 'test'
data = train_df.append([test_df, submission_df])
 print(colored('Train data set shape.......: ','yellow'), train_df.shape)
 print(colored('Test data set shape........ : ','green'), test_df.shape)
 print(colored('Submission data set shape.. : ','blue'), submission_df.shape)
 print(colored('Comibined data set shape...: ','red'), data.shape)
 print("
")
 print(colored('Unique Patient in Train data set shape.......: ','yellow'), train_df.Patient.nunique())
 print(colored('Unique Patient in Test data set shape........ : ','green'), test_df.Patient.nunique())
 print(colored('Unique Patient in Submission data set shape.. : ','blue'), submission_df.Patient.nunique())
 print(colored('Unique Patient in Comibined data set shape...: ','red'), data.Patient.nunique() )<feature_engineering>
all_data_rep_enc_1=all_data_rep_enc_1_.drop(['Utilities'],axis=1 )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
all_data_dum = pd.get_dummies(all_data_rep_enc_1,drop_first=True)
 all_data_dum.shape
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
X_train = all_data_dum[:ntrain]
x_test = all_data_dum[ntrain:]
X_train.shape
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
cv=KFold(n_splits=5,shuffle=True,random_state=241 )
COLS = ['Sex','SmokingStatus']
features = []
for col in COLS:
for mod in data[col].unique() :
features.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
def mse_cv(model):
mse_my=np.sqrt(-cross_val_score(model,X=X_train,y=y_train,scoring="neg_mean_squared_error", cv = cv))
return(mse_my.mean() )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 data['FVC_Percent'] = data['FVC'] / data['Percent']
features += ['age','percent','week','BASE']
print(features )<filter>
model=XGBRegressor(objective='reg:squarederror',n_estimators=1000)
 simp_model=mse_cv(model)
 print('Score_simple_model',simp_model )
train_df = data.loc[data.WHERE=='train']
test_df = data.loc[data.WHERE=='val']
submission_df = data.loc[data.WHERE=='test']
del data<compute_test_metric>
lean_r=[0.0005,0.005,0.05,0.1,0.2]
final_score=[]
for lr in lean_r:
model=XGBRegressor(objective='reg:squarederror',n_estimators=500, learning_rate=lr, early_stop_round=5)
 impr_score_ln_rate=mse_cv(model)
 final_score.append(impr_score_ln_rate )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric )<compute_test_metric>
cv_boost =cv_boost_ln_rate
min_err=min(cv_boost.values)
 impr_lr_rate=min_err
def get_key(cv_boost,min_err):
for k,v in cv_boost.items() :
if v==min_err:
return k,v
best_ln_rate=get_key(cv_boost,min_err)[0]
print('Impored_score_ln_rate',impr_lr_rate)
 print('Best_ln_rate',best_ln_rate )
def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v )<compute_test_metric>
n_est=[500,600,700,750,1000]
learning_rate=best_ln_rate
final_score_n_est=[]
for n in n_est:
model=XGBRegressor(objective='reg:squarederror',n_estimators=n, early_stop_round=5,learning_rate=best_ln_rate)
 impr_score=mse_cv(model)
 final_score_n_est.append(impr_score )
def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss<choose_model_class>
max_d=[2,3,4,5,6]
final_score_max_d=[]
for m in max_d:
model=XGBRegressor(objective='reg:squarederror',n_estimators=600, learning_rate=0.05, max_depth=m)
 impr_score=mse_cv(model)
 final_score_max_d.append(impr_score )
def make_model(no_feature):
z = L.Input(( no_feature,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
model=XGBRegressor(objective='reg:squarederror',n_estimators=600, learning_rate=0.05,max_depth=5)
 impr_score_3_par=mse_cv(model )
print(colored('Features for model building : ','yellow'),features)
 y = train_df['FVC'].values
X = train_df[features].values
ze = submission_df[features].values
print(colored('Training data set shape for model building : ','yellow'),X.shape)
 print(colored('Shape of submission.csv: ','yellow'),ze.shape)
 nh = X.shape[1]
print(colored('Number of features : ','yellow'),nh)
 pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( X.shape[0], 3))<train_model>
min_child=[1,2,3,4,5]
final_score_min_child=[]
for m in min_child:
model=XGBRegressor(objective='reg:squarederror',n_estimators=600, learning_rate=0.05, max_depth=5, min_child_weight=m)
 impr_score=mse_cv(model)
 final_score_min_child.append(impr_score )
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(X):
cnt += 1
print("
")
 print(colored('Fold...........: ','red'),cnt)
 net = make_model(nh)
 net.fit(X[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, validation_data=(X[val_idx], y[val_idx]), verbose=0)
 print(colored('Train.......... : ','yellow'),"train", net.evaluate(X[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print(colored('Validation............ : ','yellow'),"val", net.evaluate(X[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print(colored('Predict Validation.. : ','yellow'))
pred[val_idx] = net.predict(X[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print(colored('Predict Test...: ','yellow'))
pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
cv_boost=pd.DataFrame(( zip(min_child,final_score_min_child)) , columns=['Child','Score'] ).sort_values(by='Score',ascending=True)
 impr_score_4_par=cv_boost[:1]['Score']
impr_min_child_weight=cv_boost[:1]['Child']
print('Score XGBoost improved ln_rate,n_est,max_depth,min_child',round(impr_score_4_par,5))
print('Best_min_child',round(impr_min_child_weight))
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean)
 print(unc.min() , unc.mean() , unc.max() ,(unc>=0 ).mean() )<prepare_output>
model=XGBRegressor(objective='reg:squarederror',n_estimators=600, learning_rate=0.05,max_depth=5, min_child_weight=3)
 impr_score_4_par=mse_cv(model )
submission_df['FVC1'] = 1.*pe[:, 1]
submission_df['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = submission_df[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
subsample=[1,0.8,0.7,0.6,0.5]
final_score_subsample=[]
for m in subsample:
model=XGBRegressor(objective='reg:squarederror',n_estimators=600, learning_rate=0.05, max_depth=5, min_child_weight=3,subsample=m)
 impr_score=mse_cv(model)
 final_score_subsample.append(impr_score )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
cv_boost=pd.DataFrame(( zip(subsample,final_score_subsample)) , columns=['SubSam','Score'] ).sort_values(by='Score',ascending=True)
 impr_score_5_par=cv_boost[:1]['Score']
impr_min_subsample=cv_boost[:1]['SubSam']
print('Score XGBoost improved ln_rate,n_est,max_depth,min_child,subsample',round(impr_score_5_par,5))
print('Best_subsample',impr_min_subsample )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
model=XGBRegressor(objective='reg:squarederror',n_estimators=600, learning_rate=0.05,max_depth=5, min_child_weight=3,subsample=0.5)
 impr_score_5_par=mse_cv(model )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
print('Score XGBoost improved ln_rate,n_est,max_depth,min_child,subs',round(impr_score_5_par,5))
print('Score XGBoost impln_rate,n_est,max_dep,min_chi',round(impr_score_4_par,5))
print('Score XGBoost improved ln_rate,n_est,max_depth',round(impr_score_3_par,5))
print('Score XGBoost improved ln_rate,n_est ',round(impr_score_n_est,5))
print('Score XGBoost improved ln_rate ',round(impr_lr_rate,5))
print('Score XGBoost simple_model ',round(simp_model,5))
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
model_param_grid= XGBRegressor(objective='reg:squarederror',subsample=0.6,n_estimators=1400,learning_rate=0.03,max_depth=3,min_child_weight=3 )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
impr_score_grid=mse_cv(model_param_grid)
 print('Score XGBoost improved GridSearchCV',round(impr_score_grid,5))
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()<save_to_csv>
model = XGBRegressor(objective='reg:squarederror',learning_rate=0.03,n_estimators=1400,max_depth=3,min_child_weight=3,subsample=0.6)
 model.fit(X_train, y_train)
df.to_csv('submission.csv', index=False )<import_modules>
importance = model.feature_importances_
feat_impot=pd.DataFrame(( zip(X_train.columns, importance)) , columns=['Feat','Importance'] ).sort_values(by='Importance',ascending=False)
 feat_impot
import os
import numpy as np
import pandas as pd
import random
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import pydicom 
<define_variables>
def mse_cv_1(model,X):
mse_my=np.sqrt(-cross_val_score(model,X=X,y=y_train,scoring="neg_mean_squared_error", cv = cv))
return(mse_my.mean() )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(8675309 )<load_from_csv>
feat=[120,130,140,201]
score_feat=[]
for f in feat:
feat_best=feat_impot['Feat'][:f]
feat_best_list=list(feat_best)
 X_feat=X_train[feat_best_list]
score_feat.append(mse_cv_1(model,X_feat))
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 print(train.head())
 print(test.head())
 print(sub.head() )<merge>
feat_best=feat_impot['Feat'][:130]
feat_best_list=list(feat_best)
 X_feat=X_train[feat_best_list]
train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient")
 print(train.shape, test.shape, sub.shape )<merge>
model_feat_sel=XGBRegressor(objective='reg:squarederror',learning_rate=0.03,n_estimators=1400,max_depth=3,min_child_weight=3,subsample=0.6)
 impr_score_feat_sel=mse_cv_1(model,X_feat )
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient', 'FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb == 1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
print(FE)
 train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train.shape, test.shape, sub.shape<compute_test_metric>
print('Score XGBoost improved GridSearchCV feat selection_130 ',round(impr_score_feat_sel,5))
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return tf.keras.backend.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return tf.keras.backend.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = tf.keras.layers.Input(( nh,), name="Patient")
 x = tf.keras.layers.Dense(100, activation="relu", name="d1" )(z)
 x = tf.keras.layers.Dense(100, activation="relu", name="d2" )(x)
 p1 = tf.keras.layers.Dense(3, activation="linear", name="p1" )(x)
 p2 = tf.keras.layers.Dense(3, activation="relu", name="p2" )(x)
 preds = tf.keras.layers.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = tf.keras.models.Model(z, preds, name="definitely_not_a_CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<find_best_params>
model_boost=GradientBoostingRegressor()
score_boost=mse_cv(model_boost)
 print('Score_GradBoost',score_boost )
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
net = make_model(nh)
 print(net.summary())
 print(net.count_params() )<choose_model_class>
print('Score XGBoost improved GridSearchCV feat selection_130 ',round(impr_score_feat_sel,5))
print('Score XGBoost improved GridSearchCV ',round(impr_score_grid,5))
print('Score XGBoost improved ln_rate,n_est,max_depth,min_child,subs',round(impr_score_5_par,5))
print('Score XGBoost impln_rate,n_est,max_dep,min_chi ',round(impr_score_4_par,5))
print('Score XGBoost improved ln_rate,n_est,max_depth',round(impr_score_3_par,5))
print('Score XGBoost improved ln_rate,n_est ',round(impr_score_n_est,5))
print('Score XGBoost improved ln_rate ',round(impr_lr_rate,5))
print('Score XGBoost improved simple_model ',round(simp_model,5))
print('Score GradientBoost ',round(score_boost,5))
NFOLD = 5
kf = KFold(n_splits=NFOLD )<train_model>
model=XGBRegressor(objective='reg:squarederror',learning_rate=0.03,n_estimators=1400,
max_depth=3,min_child_weight=3,subsample=0.6)
 X_train_feat=X_train[feat_best_list]
x_test_feat=x_test[feat_best_list]
XGB=model.fit(X_train_feat,y_train)
 test=x_test_feat.copy()
finalMd = np.expm1(XGB.predict(test))
finalMd
%%time
cnt = 0
EPOCHS = 800
BATCH_SIZE = 64
diff_sum = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 train_loss, train_score = net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE)
 print(f"Train Loss: {train_loss} Score: {train_score}")
 val_loss, val_score = net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE)
 print(f"Val Loss: {val_loss} Score: {val_score}")
 score_diff = val_score - train_score
diff_sum += score_diff
print(f"Score diff: {score_diff}")
 print("Predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("Predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print(f"Score diff sum : {diff_sum}" )<feature_engineering>
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 sample_submission.iloc[:,1] = finalMd
sample_submission.to_csv("sample_submission.csv", index=False )