Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<load_from_csv>
warnings.filterwarnings("ignore")
train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample_sub=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )<filter>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
train[train.SalePrice>300000]<sort_values>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
corr_dict=correlation_train['SalePrice'].sort_values(ascending=False ).to_dict()
important_columns=[]
for key,value in corr_dict.items() :
if(( value>0.1)&(value<0.8)) |(value<=-0.1):
important_columns.append(key)
 important_columns<concatenate>
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )
train_test=pd.concat([train,test],axis=0,sort=False)
 train_test.head()<create_dataframe>
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub] )
pd.set_option('display.max_rows', 5000)
 train_test_null_info=pd.DataFrame(train_test.isnull().sum() ,columns=['Count of NaN'])
 train_test_dtype_info=pd.DataFrame(train_test.dtypes,columns=['DataTypes'])
 train_tes_info=pd.concat([train_test_null_info,train_test_dtype_info],axis=1)
 train_tes_info<data_type_conversions>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
train_test.loc[train_test['Fireplaces']==0,'FireplaceQu']='Nothing'
train_test['LotFrontage'] = train_test['LotFrontage'].fillna(train_test.groupby('1stFlrSF')['LotFrontage'].transform('mean'))
train_test['LotFrontage'].interpolate(method='linear',inplace=True)
 train_test['LotFrontage']=train_test['LotFrontage'].astype(int)
 train_test['MasVnrArea'] = train_test['MasVnrArea'].fillna(train_test.groupby('MasVnrType')['MasVnrArea'].transform('mean'))
train_test['MasVnrArea'].interpolate(method='linear',inplace=True)
 train_test['MasVnrArea']=train_test['MasVnrArea'].astype(int)
 train_test["Fence"] = train_test["Fence"].fillna("None")
 train_test["FireplaceQu"] = train_test["FireplaceQu"].fillna("None")
 train_test["Alley"] = train_test["Alley"].fillna("None")
 train_test["PoolQC"] = train_test["PoolQC"].fillna("None")
 train_test["MiscFeature"] = train_test["MiscFeature"].fillna("None")
 train_test.loc[train_test['BsmtFinSF1']==0,'BsmtFinType1']='Unf'
train_test.loc[train_test['BsmtFinSF2']==0,'BsmtQual']='TA'
train_test['YrBltRmd']=train_test['YearBuilt']+train_test['YearRemodAdd']
train_test['Total_Square_Feet'] =(train_test['BsmtFinSF1'] + train_test['BsmtFinSF2'] + train_test['1stFlrSF'] + train_test['2ndFlrSF'] + train_test['TotalBsmtSF'])
 train_test['Total_Bath'] =(train_test['FullBath'] +(0.5 * train_test['HalfBath'])+ train_test['BsmtFullBath'] +(0.5 * train_test['BsmtHalfBath']))
train_test['Total_Porch_Area'] =(train_test['OpenPorchSF'] + train_test['3SsnPorch'] + train_test['EnclosedPorch'] + train_test['ScreenPorch'] + train_test['WoodDeckSF'])
 train_test['exists_pool'] = train_test['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_garage'] = train_test['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_fireplace'] = train_test['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_bsmt'] = train_test['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 train_test['old_house'] = train_test['YearBuilt'].apply(lambda x: 1 if x <1990 else 0)
 for i in train_test.columns:
if 'SalePrice' not in i:
if 'object' in str(train_test[str(i)].dtype):
train_test[str(i)]=train_test[str(i)].fillna(method='ffill')
<categorify>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
columns =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1', 'YrSold', 'MoSold', 'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope', 'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond')
 for col in columns:
lbl_enc = LabelEncoder()
lbl_enc.fit(list(train_test[col].values))
train_test[col] = lbl_enc.transform(list(train_test[col].values))<sort_values>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
numeric_features = train_test.dtypes[train_test.dtypes != "object"].index
skewed_features = train_test[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 print(skewed_features )<filter>
lb =LabelEncoder()
data["Sex"] = lb.fit_transform(data["Sex"] )
high_skewness = skewed_features[abs(skewed_features)> 0.9]
skewed_features = high_skewness.index<train_model>
FE = []
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['Sex','age','percent','week','BASE']
print(high_skewness)
 print('
Variables with high skewness:
',skewed_features )<feature_engineering>
range_n_clusters = [2,3,4,5,6,7,8,9]
for num_clusters in range_n_clusters:
kmeans = KMeans(n_clusters=num_clusters, max_iter=90)
 kmeans.fit(features_scaled)
 cluster_labels = kmeans.labels_
silhouette_avg = silhouette_score(features_scaled, cluster_labels)
 print("For n_clusters={0}, the silhouette score is {1}".format(num_clusters, silhouette_avg))
for feature in skewed_features:
train_test[feature] = boxcox1p(train_test[feature], boxcox_normmax(train_test[feature] + 1)) 
<categorify>
kmeans = KMeans(n_clusters=2, max_iter=3000)
 kmeans.fit(features_scaled)
 features_scaled["Cluster_Id"] =kmeans.labels_
features_scaled["FVC"] =data["FVC"]
train_test=pd.get_dummies(train_test,dtype='int8' )<create_dataframe>
algorithm =(KMeans(n_clusters =2,init='k-means++', n_init = 10 ,max_iter=10000,
tol=0.0001, random_state= 111 , algorithm='elkan'))
algorithm.fit(features_scaled)
 data['Cluster_Id'] = algorithm.labels_
del features_scaled
train_test_null_info=pd.DataFrame(train_test.isnull().sum() ,columns=['Count of NaN'])
 train_test_dtype_info=pd.DataFrame(train_test.dtypes,columns=['DataTypes'])
 train_test_info=pd.concat([train_test_null_info,train_test_dtype_info],axis=1)
 train_test_info<split>
COLS = ['SmokingStatus']
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
train=train_test[0:1460]
test=train_test[1460:2919]<sort_values>
data = pd.get_dummies(data,columns=["Cluster_Id"])
 for col in("Cluster_Id_0","Cluster_Id_1"):
data[col] = data[col].astype(int)
 FE+=["Cluster_Id_0","Cluster_Id_1"]
corr_dict2=corr_new_train['SalePrice'].sort_values(ascending=False ).to_dict()
corr_dict2<define_variables>
features_scaled =data[["min_FVC","percent","BASE","age"]]
inertia = []
algorithm =(DBSCAN(eps=0.6, metric='euclidean',
metric_params=None, algorithm="auto", leaf_size=20, p=None, n_jobs=None))
algorithm.fit(features_scaled)
 labels = algorithm.labels_
core_samples_mask = np.zeros_like(labels, dtype=bool)
 core_samples_mask[algorithm.core_sample_indices_] = True
n_clusters_ = len(set(labels)) -(1 if -1 in labels else 0)
 n_noise_ = list(labels ).count(-1)
 print("Estimated number of clusters: %d" % n_clusters_)
 print("Estimated number of noise points: %d" % n_noise_)
 s = metrics.silhouette_score(features_scaled, algorithm.labels_)
 print(f"Silhouette Coefficient for the Dataset Clusters: {s:.2f}" )
best_columns=[]
for key,value in corr_dict2.items() :
if(( value>=0.3175)&(value<0.9)) |(value<=-0.315):
best_columns.append(key)
 best_columns<feature_engineering>
algorithm =(DBSCAN(eps=0.5, min_samples=1, metric='euclidean',
metric_params=None, algorithm="auto", leaf_size=20, p=None, n_jobs=None))
algorithm.fit(features_scaled)
 labels = algorithm.labels_
train['SalePrice_Log1p'] = np.log1p(train.SalePrice )<filter>
data['DBSCAN_CLUSTER'] = algorithm.labels_
for mod in data["DBSCAN_CLUSTER"].unique() :
FE.append(mod)
 data[mod] =(data["DBSCAN_CLUSTER"] == mod ).astype(int)
print(len(train[(train.KitchenQual==3)&(train.SalePrice>360000)].index))<train_on_grid>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
dbscan = DBSCAN(eps=1400, min_samples=20 ).fit(train_pca)
 core_samples_mask = np.zeros_like(dbscan.labels_, dtype=bool)
 core_samples_mask[dbscan.core_sample_indices_] = True
labels=dbscan.labels_
n_clusters_ = len(set(labels)) -(1 if -1 in labels else 0 )<create_dataframe>
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
labels=pd.DataFrame(labels,columns=['Classes'])
 print(labels[labels['Classes']==-1] )<concatenate>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def make_model(learning_rate):
z = L.Input(( 178,), name="Patient")
 x = L.Dense(200, activation="relu",use_bias=True,kernel_regularizer=regularizers.l2(1e-7),
bias_regularizer=regularizers.l2(1e-7),
activity_regularizer=regularizers.l1(1e-7))(z)
 keras.layers.Dropout(0.1)
 x = L.Dense(100, activation="relu",use_bias=True,kernel_regularizer=regularizers.l2(1e-7),
bias_regularizer=regularizers.l2(1e-7),
activity_regularizer=regularizers.l1(1e-7))(x)
 keras.layers.Dropout(0.1)
 x = L.Dense(10, activation="relu",use_bias=True )(x)
 keras.layers.Dropout(0.1)
 p1 = L.Dense(3, activation="linear", name="p1",use_bias=True )(x)
 p2 = L.Dense(3, activation="relu", name="p2",use_bias=True )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
train=pd.concat([train,labels],axis=1 )<filter>
es_callback = keras.callbacks.EarlyStopping(monitor='val_loss', patience=15)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', factor=0.5, patience=5, verbose=0, mode='auto',
min_delta=0.0001, cooldown=0, min_lr=0.0001 
)
train[train.Classes==-1]<drop_column>
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 cnt = 0
BATCH_SIZE = 16
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(learning_rate=0.04)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=1000,
validation_data=(z[val_idx], y[val_idx]),callbacks=[es_callback,reduce_lr], verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
train.drop([197,810,1170,1182,1298,1386,1423],axis=0,inplace=True )<drop_column>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
"Looking at the SalePrice and SalePrice_Log1p visualizations"
train = train[train.GarageArea * train.GarageCars < 3700]
train = train[(train.FullBath +(train.HalfBath*0.5)+ train.BsmtFullBath +(train.BsmtHalfBath*0.5)) <5]<count_missing_values>
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
train.isnull().sum()<count_missing_values>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
train.isnull().sum()<count_missing_values>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
test.isnull().sum()<count_missing_values>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<drop_column><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )