Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<prepare_x_and_y>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
features = ['year', 'hour','month', 'distance','passenger_count', 'jfk_dist','ewr_dist','lga_dist','ctr_pick_dist','ctr_drop_dist']
X = df_train[features].values
y = df_train['fare_amount'].values<feature_engineering>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
df_test['year'] = df_test.pickup_datetime.apply(lambda t: t.year)
 df_test['month'] = df_test.pickup_datetime.apply(lambda t: t.month)
 df_test['hour'] = df_test.pickup_datetime.apply(lambda t: t.hour)
 df_test['distance'] = distance(df_test.pickup_latitude, df_test.pickup_longitude, \
df_test.dropoff_latitude, df_test.dropoff_longitude)
 df_test = add_all_dist(df_test )<prepare_x_and_y>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
X_kaggle_test = df_test[features].values<train_model>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=265
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=10,test_size=0.3)
 def XGBmodel(x_train,x_test,y_train,y_test):
matrix_train = xgb.DMatrix(x_train,label=y_train)
 matrix_test = xgb.DMatrix(x_test,label=y_test)
 model=xgb.train(params={'objective':'reg:linear','eval_metric':'rmse'},
dtrain=matrix_train,num_boost_round=200,
early_stopping_rounds=100,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(X_train,X_test,y_train,y_test)
 prediction = model.predict(xgb.DMatrix(X_kaggle_test), ntree_limit = model.best_ntree_limit)
<save_to_csv>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
submission = pd.DataFrame({
"key": df_test['key'],
"fare_amount": prediction.round(2)
 })
 submission.to_csv('submission.csv',index=False)
 submission 
<load_from_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
df_train=pd.read_csv(".. /input/train.csv", sep=',', lineterminator='
',infer_datetime_format=True,nrows=50000000,
chunksize=1000000)
 df_test=pd.read_csv(".. /input/test.csv", sep=',', lineterminator='
')
 df_test.rename(columns={'passenger_count\r':'passenger_count'}, inplace=True)
<compute_test_metric>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
def getDistance(lat1,lon1,lat2,lon2):
R = 6373.0
dlon = lon2 - lon1
dlat = lat2 - lat1
a =(np.sin(dlat/2)) **2 + np.cos(lat1)* np.cos(lat2)*(np.sin(dlon/2)) **2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
distance = R * c
return distance
def clean_df(df):
return df[(df.fare_amount > 0)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -70)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.dropoff_longitude > -80)&(df.dropoff_longitude < -70)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.passenger_count > 0)&(df.passenger_count < 10)] 
<load_from_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
df_test=pd.read_csv(".. /input/test.csv", sep=',', lineterminator='
')
 df_test.rename(columns={'passenger_count\r':'passenger_count'}, inplace=True)
 df_test_2=df_test
df_test_2["distance"]=getDistance(df_test_2.pickup_latitude,df_test_2.pickup_longitude,df_test_2.dropoff_latitude,df_test_2.dropoff_longitude)
 print("test distance completed")
 df_test_2['date_pickup']=pd.to_datetime(df_test_2['pickup_datetime'])
 print("test conversion completed")
 WeekValue_dict = {0:0,1: 0, 2: 0, 3: 0, 4: 0,5: 1, 6: 1}
df_test_2['WeekValue'] = df_test_2['date_pickup'].dt.dayofweek.map(WeekValue_dict)
 print("test WeekValue completed")
 df_test_2['year'] = df_test_2['date_pickup'].dt.year
df_test_2['month'] = df_test_2['date_pickup'].dt.month
df_test_2['day'] = df_test_2['date_pickup'].dt.day
df_test_2['hour'] = df_test_2['date_pickup'].dt.hour
print(" test dates completed")
 day_int_dict = {6:0,7:0,8:0,9:0,10:0,11:0,12:1,13:1,14:2,15:2,16:2,17:2,18:2,0:4,1:4,2:4,3:4,4:4,5:4,19:3,
20:3,21:3,22:3,23:3}
df_test_2['day_int'] = df_test_2['hour'].map(day_int_dict)
 df_test_2['day_name'] = df_test_2['date_pickup'].dt.day_name()
season_dict = {12:0,1: 0, 2: 0, 3: 1, 4: 1,5: 1, 6: 2, 7: 2, 8: 2,9: 3, 10: 3, 11: 3}
df_test_2['season'] = df_test_2['month'].map(season_dict)
 print("test 3rd completed")
 lb_make = LabelEncoder()
df_test_2["day_name_num"] = lb_make.fit_transform(df_test_2["day_name"])
 df_test_3=df_test_2[['passenger_count', 'distance', 'WeekValue', 'year','month','day_int','season','day_name_num',"key",'pickup_longitude',
'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']] 
<train_model>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
chunk_list = []
df_train_4 = pd.DataFrame(columns=['passenger_count', 'distance', 'WeekValue', 'year', 'day_int', 'season', 'day_name_num','pickup_longitude',
'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude','fare_amount'])
 params ={
'n_estimators':100,
'max_depth':8,
'eta':0.01,
}
model = xgb.XGBRegressor(params=params)
 chunk_idx = 0
for chunk in df_train:
chunk.rename(columns={'passenger_count\r':'passenger_count'}, inplace=True)
 have_pass = chunk['passenger_count']>0
have_money= chunk['fare_amount']>0
df_train_2 = chunk[have_pass & have_money]
df_train_2 = clean_df(df_train_2)
 print("Clean data completed")

df_train_2["distance"]=getDistance(df_train_2.pickup_latitude,df_train_2.pickup_longitude,df_train_2.dropoff_latitude,df_train_2.dropoff_longitude)
 print("train distance completed")
 df_train_2['date_pickup']=pd.to_datetime(df_train_2['pickup_datetime'],format='%Y-%m-%d %H:%M:%S UTC')
 print("train conversion completed")
 WeekValue_dict = {0:0,1: 0, 2: 0, 3: 0, 4: 0,5: 1, 6: 1}
df_train_2['WeekValue'] = df_train_2['date_pickup'].dt.dayofweek.map(WeekValue_dict)
 print("train WeekValue completed")
 df_train_2['year'] = df_train_2['date_pickup'].dt.year
df_train_2['month'] = df_train_2['date_pickup'].dt.month
df_train_2['day'] = df_train_2['date_pickup'].dt.day
df_train_2['hour'] = df_train_2['date_pickup'].dt.hour
print(" train dates completed")
 day_int_dict = {6:0,7:0,8:0,9:0,10:0,11:0,12:1,13:1,14:2,15:2,16:2,17:2,18:2,0:4,1:4,2:4,3:4,4:4,5:4,19:3,
20:3,21:3,22:3,23:3}
df_train_2['day_int'] = df_train_2['hour'].map(day_int_dict)
 df_train_2['day_name'] = df_train_2['date_pickup'].dt.day_name()
season_dict = {12:0,1: 0, 2: 0, 3: 1, 4: 1,5: 1, 6: 2, 7: 2, 8: 2,9: 3, 10: 3, 11: 3}
df_train_2['season'] = df_train_2['month'].map(season_dict)
 print("train 3rd completed")
 lb_make = LabelEncoder()
df_train_2["day_name_num"] = lb_make.fit_transform(df_train_2["day_name"])
 if chunk_idx > 0:
df_train_4.append(df_train_2)
 else:
df_train_4=df_train_2
lister_4=['passenger_count', 'distance', 'WeekValue', 'year', 'day_int', 'season', 'day_name_num','pickup_longitude',
'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']
lister_5=['fare_amount']
X_train, X_test, y_train, y_test= train_test_split(df_train_4[lister_4],df_train_4[lister_5], test_size=0.1,
random_state=42)
 if chunk_idx > 0:
model.fit(X_train, y_train, xgb_model='model_1.model')
 model.save_model('model_1.model')
 else:
model.fit(X_train, y_train)
 model.save_model('model_1.model')
 chunk_idx = chunk_idx + 1
rmse = sqrt(mean_squared_error(y_test, model.predict(X_test)))
 print("RMSE RESULT ",rmse)
 accuracy = r2_score(y_test, model.predict(X_test))
print("Accuracy: %.2f%%" %(accuracy * 100.0))
print(1)
 predictions = model.predict(df_test_3[lister_4])
 my_submission = pd.DataFrame({ 'key': df_test_3.key,'fare_amount': predictions.round(2)})
 print("here")
<save_to_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
my_submission.to_csv('sample_submission.csv', index=False)
 print("Writing complete")
<set_options>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
!pip install --upgrade seaborn
np.random.seed(42)
 plt.rcParams.update({'figure.max_open_warning': 0})
 pd.options.mode.chained_assignment = None
pd.set_option('display.max_rows', 250)
 pd.set_option('display.max_columns', 250 )<import_modules>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
!pip install --upgrade lightgbm 
<load_from_csv>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train_id = train.Id
test_id = test.Id
print("Train set: ", train.shape)
 print("Test set: ", test.shape)
 train.info()<concatenate>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(500, activation="elu", name="d1" )(z)
 x = L.Dense(500, activation="elu", name="d2" )(x)
 p1 = L.Dense(3, activation="elu", name="p1" )(x)
 p2 = L.Dense(3, activation="elu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer="adam", metrics=[score])
 return model
df = pd.concat([train, test])
 df<count_values>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
def col_types(df):
num_cols = df.loc[:, df.dtypes != "object"].columns.tolist()
cat_cols = df.loc[:, df.dtypes == "object"].columns.tolist()
ord_cols = []
for col in num_cols:
if df[col].value_counts().size < 20:
ord_cols.append(col)
 num_cols = [x for x in num_cols if x not in ord_cols + ["Id", "SalePrice"]]
return num_cols, cat_cols, ord_cols<count_missing_values>
NFOLD = 3
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
def missing(df):
miss = pd.DataFrame({"no_missing_values": df.isnull().sum() ,
"missing_value_ratio":(df.isnull().sum() / df.shape[0] ).round(4),
"missing_in_train": df[df.SalePrice.notnull() ].isnull().sum() ,
"missing_in_test": df[df.SalePrice.isnull() ].isnull().sum() })
 return miss[miss.no_missing_values > 0].sort_values("no_missing_values", ascending=False)
 missing(df )<filter>
NFOLD = 2
kf = KFold(n_splits=NFOLD )
df[(df.GarageFinish.isnull())&(df.GarageType.notnull())]<feature_engineering>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=1000,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
df.loc[(df.GarageFinish.isnull())&(df.GarageType.notnull()), "GarageFinish"] = "Fin"
df.loc[(df.GarageCars.isnull())&(df.GarageType.notnull()), "GarageCars"] = 1
df.loc[(df.GarageQual.isnull())&(df.GarageType.notnull()), "GarageQual"] = "TA"
df.loc[(df.GarageCond.isnull())&(df.GarageType.notnull()), "GarageCond"] = "TA"<filter>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
df[df.GarageYrBlt == df.YearBuilt].shape[0]<feature_engineering>
NFOLD = 2
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
df.loc[(df.GarageYrBlt.isnull())&(df.GarageType.notnull()), "GarageYrBlt"] = df.loc[(df.GarageYrBlt.isnull())&(df.GarageType.notnull())].YearBuilt<define_variables>
NFOLD = 2
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
df[(df.GarageType == "Detchd")&(df.YearBuilt < 1930)&(df.YearRemodAdd < 2000)&(df.YearRemodAdd > 1980)&(df.GarageCars == 1)].GarageArea.median()<feature_engineering>
NFOLD = 4
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
df.loc[(df.GarageArea.isnull())&(df.GarageType.notnull()), "GarageArea"] = 234<define_variables>
NFOLD = 10
kf = KFold(n_splits=NFOLD)
 cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
num_cols, cat_cols, ord_cols = col_types(df )<correct_missing_values>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
none_cols = ["PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu", "GarageCond", "GarageFinish", "GarageQual",
"GarageType", "BsmtExposure", "BsmtCond", "BsmtQual", "BsmtFinType1", "MasVnrType", "BsmtFinType2"]
for col in none_cols:
df[col].fillna("None", inplace=True)
 missing(df )<feature_engineering>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
df.loc[(df.MasVnrArea.isnull())&(df.MasVnrType == "None"), "MasVnrArea"] = 0<data_type_conversions>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
for col in ["BsmtFullBath", "BsmtHalfBath", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF", "GarageYrBlt"]:
df[col].fillna(0, inplace = True)
 df["TotalBsmtSF"].fillna(df["BsmtFinSF1"] + df["BsmtFinSF2"] + df["BsmtUnfSF"], inplace = True)
 missing(df )<count_values>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<groupby><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )