Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<load_from_csv>
warnings.filterwarnings("ignore" )
test_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv" )<data_type_conversions>
import torch.nn as nn
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
test_df["Date"] = pd.to_datetime(test_df["Date"] )<feature_engineering>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = True
seed_everything(42 )
test_df['day'] = test_df['Date'].dt.day<feature_engineering>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
device = torch.device('cuda' )
test_df["month"] = test_df["Date"].dt.month<drop_column>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
test_df.drop('Date',axis=1,inplace=True)
 test_df.head()<feature_engineering>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
test_df.fillna("nan",inplace=True)
 test_df["Province_State"] = test_df.apply(lambda x: province(x["Province_State"],x["Country_Region"]),axis=1 )<categorify>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
test_df["Country_Region"] = labelencoder(test_df["Country_Region"].values)
 test_df["Province_State"] = labelencoder(test_df["Province_State"].values )<import_modules>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import MinMaxScaler<prepare_x_and_y>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
poly_reg_cc = PolynomialFeatures(degree = 4)
 poly_reg_ft = PolynomialFeatures(degree = 4)
 reg_cc = LinearRegression()
reg_ft = LinearRegression()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = train_df[train_df["Country_Region"]==country]["Province_State"].unique()
for state in states:
train_df_filt = train_df[(train_df["Country_Region"]==country)&(train_df["Province_State"]==state)]
y_train_cc = train_df_filt["ConfirmedCases"].values
y_train_ft = train_df_filt["Fatalities"].values
X_train = train_df_filt[["month","day"]]
test_df_filt = test_df[(test_df["Country_Region"]==country)&(test_df["Province_State"]==state)]
X_test = test_df_filt.drop('ForecastId',axis=1)
 X_test = X_test[["month","day"]]
test_Id = test_df_filt["ForecastId"].values
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 scaler_cc = MinMaxScaler()
scaler_ft = MinMaxScaler()
y_train_cc=y_train_cc.reshape(-1,1)
 y_train_ft=y_train_ft.reshape(-1,1)
 y_train_cc=scaler_cc.fit_transform(y_train_cc)
 y_train_ft=scaler_ft.fit_transform(y_train_ft)
 y_train_cc = y_train_cc.flatten()
y_train_ft = y_train_ft.flatten()
X_train_poly = poly_reg_cc.fit_transform(X_train)
 reg_cc.fit(X_train_poly,y_train_cc)
 X_test_poly = poly_reg_cc.fit_transform(X_test)
 test_cc = reg_cc.predict(X_test_poly)
 test_cc = test_cc.reshape(-1,1)
 test_cc = scaler_cc.inverse_transform(test_cc)
 test_cc = test_cc.flatten()
X_train_poly = poly_reg_ft.fit_transform(X_train)
 reg_ft.fit(X_train_poly,y_train_ft)
 X_test_poly = poly_reg_ft.fit_transform(X_test)
 test_ft = reg_ft.predict(X_test_poly)
 test_ft = test_ft.reshape(-1,1)
 test_ft = scaler_ft.inverse_transform(test_ft)
 test_ft = test_ft.flatten()
df = pd.DataFrame({'ForecastId': test_Id, 'ConfirmedCases': test_cc, 'Fatalities': test_ft})
 df_out = pd.concat([df_out, df], axis=0 )<data_type_conversions>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
df_out["Fatalities"] = df_out["Fatalities"].apply(int)
 df_out["ConfirmedCases"] = df_out["ConfirmedCases"].apply(int)
 df_out[:10]<save_to_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
df_out["ForecastId"] = df_out["ForecastId"].astype('int32')
 df_out["Fatalities"] = df_out["Fatalities"].astype('int32')
 df_out["ConfirmedCases"] = df_out["ConfirmedCases"].astype('int32')
 df_out.info()
df_out.to_csv("submission.csv",index=False)
 sub = pd.read_csv("submission.csv")
 sub[:20]<feature_engineering>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 train["Province_State"] = train["Province_State"].fillna('')
 test["Province_State"] = test["Province_State"].fillna('')
 train["Month"], train["Day"] = 0, 0
for i in range(len(train)) :
train["Month"][i] =(train["Date"][i] ).split("-")[1]
train["Day"][i] =(train["Date"][i] ).split("-")[2]
test["Month"], test["Day"] = 0, 0
for i in range(len(test)) :
test["Month"][i] =(test["Date"][i] ).split("-")[1]
test["Day"][i] =(test["Date"][i] ).split("-")[2]<drop_column>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
for i in range(len(train)) :
if train["Province_State"][i] != '':
train["Country_Region"][i] = str(train["Province_State"][i])+ "(" + str(train["Country_Region"][i])+ ")"
for i in range(len(test)) :
if test["Province_State"][i] != '':
test["Country_Region"][i] = str(test["Province_State"][i])+ "(" + str(test["Country_Region"][i])+ ")"
train.drop(columns = "Province_State", inplace=True)
 test.drop(columns = "Province_State", inplace=True)
 train.rename(columns = {"Country_Region" : "Country/State"}, inplace=True)
 test.rename(columns = {"Country_Region" : "Country/State"}, inplace=True)
<feature_engineering>
class MishFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x)
 return x * torch.tanh(F.softplus(x))
@staticmethod
def backward(ctx, grad_output):
x = ctx.saved_variables[0]
sigmoid = torch.sigmoid(x)
 tanh_sp = torch.tanh(F.softplus(x))
return grad_output *(tanh_sp + x * sigmoid *(1 - tanh_sp * tanh_sp))
class Mish(nn.Module):
def forward(self, x):
return MishFunction.apply(x)
 def to_Mish(model):
for child_name, child in model.named_children() :
if isinstance(child, nn.ReLU):
setattr(model, child_name, Mish())
 else:
to_Mish(child )
i = 0
for value in train["Country/State"].unique() :
if i < len(train):
j = 1
while(train["Country/State"][i] == value):
train["Day"][i] = j
j += 1; i += 1
if i == len(train):
break
i = 0
for value in test["Country/State"].unique() :
if i < len(test):
j = 72
while(test["Country/State"][i] == value):
test["Day"][i] = j
j += 1; i += 1
if i == len(test):
break<drop_column>
C1, C2 = torch.tensor(70,dtype=torch.float),torch.tensor(1000,dtype=torch.float)
 C1, C2 = C1.to(device),C2.to(device)
 def score(y_true, y_pred):
y_true = y_true.to(torch.float)
 y_pred = y_pred.to(torch.float)
 sigma = y_pred[:,2] - y_pred[:,0]
fvc_pred = y_pred[:,1]
sigma_clip = torch.max(sigma, C1)
 delta = torch.abs(y_true[:,0] - fvc_pred)
 delta = torch.min(delta, C2)
 sq2 = torch.sqrt(torch.tensor(2, dtype=torch.float))
metric =(delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)
 return torch.mean(metric)
 def qloss(y_true, y_pred):
device = y_true.device
qs = [0.2, 0.50, 0.8]
q = torch.tensor(np.array([qs]), dtype=torch.float32)
 q = q.to(device)
 e = y_true - y_pred
v = torch.max(q*e,(q-1)*e)
 return torch.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
class make_model(nn.Module):
def __init__(self, in_ch, out_ch=3):
super(make_model, self ).__init__()
self.fc1 = nn.Sequential(
 nn.Linear(in_ch, 100),
Mish() 
)
 self.fc2 = nn.Sequential(
 nn.Linear(100, 100),
Mish() 
)
 self.fc3_p1 = nn.Linear(100, out_ch)
 self.fc3_p2 = nn.Sequential(
 nn.Linear(100, out_ch),
Mish() 
)
 def forward(self, x):
x = self.fc1(x)
 x = self.fc2(x)
 x1 = self.fc3_p1(x)
 x2 = self.fc3_p2(x)
 x = x1 + torch.cumsum(x2,dim=1)
 return x
train = train.drop(columns = ["Date"])
 test = test.drop(columns = ["Date"] )<create_dataframe>
Mloss = mloss(0.8)
 y_true =(torch.tensor([2100,2300])).to(device ).reshape(2,1)
 pred =(torch.tensor([[1800,2100,2400],
[2100,2300,2600]])).to(device ).reshape(2,3)
 Mloss(y_true,pred )
poly_reg_cc = PolynomialFeatures(degree = 4)
 poly_reg_ft = PolynomialFeatures(degree = 4)
 reg_cc = LinearRegression()
reg_ft = LinearRegression()
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for value in countries:
train_temp = train.loc[train["Country/State"] == value]
test_temp = test.loc[test["Country/State"] == value]
train_temp_cc = train_temp["ConfirmedCases"].loc[train["Country/State"] == value].to_frame()
train_temp_ft = train_temp["Fatalities"].loc[train["Country/State"] == value].to_frame()
train_temp_X = train_temp.iloc[:, 4:6]
test_temp_X = test_temp.iloc[:, 2:4]
sc1 = StandardScaler()
train_temp_X = sc1.fit_transform(train_temp_X)
 test_temp_X = sc1.transform(test_temp_X)
 sc_cc = StandardScaler()
sc_ft = StandardScaler()
train_temp_cc = sc_cc.fit_transform(train_temp_cc)
 train_temp_ft = sc_ft.fit_transform(train_temp_ft)
 X_poly = poly_reg_cc.fit_transform(train_temp_X)
 reg_cc.fit(X_poly, train_temp_cc)
 test_cc = sc_cc.inverse_transform(reg_cc.predict(poly_reg_cc.fit_transform(test_temp_X)))
 X_poly = poly_reg_ft.fit_transform(train_temp_X)
 reg_ft.fit(X_poly, train_temp_ft)
 test_ft = sc_ft.inverse_transform(reg_ft.predict(poly_reg_ft.fit_transform(test_temp_X)))
 a = int(train["Day"].loc[train["Country/State"] == "India"].max())
 b = int(a - test_temp["Day"].min())
 test_cc[0:b+1] = sc_cc.inverse_transform(train_temp_cc)[(a-b-1):(a)]
test_ft[0:b+1] = sc_ft.inverse_transform(train_temp_ft)[(a-b-1):(a)]
test_cc = test_cc.flatten()
test_ft = test_ft.flatten()
sub_temp = pd.DataFrame({'ForecastId': test_temp["ForecastId"].loc[test["Country/State"] == value],
'ConfirmedCases': test_cc, 'Fatalities': test_ft})
 sub = pd.concat([sub, sub_temp], axis = 0 )<data_type_conversions>
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 pd_patient = pd.DataFrame({"Patient":tr["Patient"].unique() })
 for idx,(tr_idx, val_idx)in enumerate(kf.split(pd_patient)) :
pd_patient.loc[val_idx,"fold"] = idx
pd_patient.head()
sub["ForecastId"] = sub["ForecastId"].astype('int32' )<save_to_csv>
tr['fold'] = -1
for i in range(len(pd_patient)) :
tr.fold[tr.Patient==pd_patient.loc[i,"Patient"]] = pd_patient.loc[i,"fold"]
tr.head()
sub.to_csv("submission.csv", index = False )<load_from_csv>
ze =(sub[FE].values ).astype(np.float32)
 pe = np.zeros(( ze.shape[0], 3))
batch = 128
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 print(df.shape,"
",df.head() )<feature_engineering>
class Data_Generate(Dataset):
def __init__(self,data,label=None):
self.data = data
self.label = label
def __getitem__(self,index):
z_ = self.data[index]
if self.label is not None:
y_ = self.label[index]
y_ = y_[None,]
return z_,y_
else:
return z_
def __len__(self):
return len(self.data )
df["Province_State"].fillna("state", inplace = True)
 df["Country_Region"] = [country_name.replace("'","")for country_name in df["Country_Region"]]
print(df.shape,"
",df.head() )<split>
test_db = Data_Generate(ze)
 test_loader = DataLoader(test_db, batch_size=batch, shuffle=False, num_workers=4 )
X_y=shuffle(new_data)
 y_cases=X_y['target_infection']
y_fatal=X_y['target_fatal']
X=X_y.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases, X_test_cases, y_train_cases, y_test_cases = train_test_split(X, y_cases, test_size=0.33)
 X_train_fatal, X_test_fatal, y_train_fatal, y_test_fatal = train_test_split(X, y_fatal, test_size=0.33)
 print("Shape of infection train dataset:",(X_train_cases.shape,y_train_cases.shape))
print("Shape of infection test dataset:",(X_test_cases.shape,y_test_cases.shape))
print("Shape of fatal train dataset:",(X_train_fatal.shape,y_train_fatal.shape))
print("Shape of fatal test dataset:",(X_test_fatal.shape,y_test_fatal.shape))<train_on_grid>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
self.delta = delta
self.path = path
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score + self.delta:
self.counter += 1
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , self.path)
 self.val_loss_min = val_loss
reg_case=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2700)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_case, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_cases, y_train_cases)
 results = pd.DataFrame.from_dict(clf.cv_results_ )<train_model>
%%time
cnt = 0
EPOCHS = 1000
criterion = mloss(0.8)
 list_train_loss,list_val_loss,list_train_score,list_val_score = [],[],[],[]
for fold in range(NFOLD):
val_out = []
print(f"FOLD {fold+1}")
 tr_z = tr[FE][tr.fold!=fold].values.astype(np.float32)
 tr_y = tr.FVC[tr.fold!=fold].values.astype(np.float32)
 val_z = tr[FE][tr.fold==fold].values.astype(np.float32)
 val_y = tr.FVC[tr.fold==fold].values.astype(np.float32)
 train_db = Data_Generate(tr_z,tr_y)
 train_loader = DataLoader(train_db, batch_size=batch, shuffle=True, num_workers=4)
 val_db = Data_Generate(val_z,val_y)
 val_loader = DataLoader(val_db, batch_size=batch, shuffle=False, num_workers=4)
 tr_num_batch = len(train_loader)
 val_num_batch = len(val_loader)
 net = make_model(in_ch=len(FE)).to(device)
 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001, weight_decay=5e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=20, min_lr=1e-8, verbose=False)
 early_stopping = EarlyStopping(patience=80,path=f'Osic-NN-fold_{fold}.pth')
 for epoch in tqdm(range(EPOCHS)) :
train_loss,train_score,val_loss,val_score = 0,0,0,0
net.train()
for idx, sample in enumerate(train_loader):
data, label = sample
data, label = data.to(device), label.to(device)
 out = net(data)
 loss = criterion(label, out)
 score_ = score(label ,out)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss += loss.item() /tr_num_batch
train_score += score_.item() /tr_num_batch
list_train_loss.append(train_loss)
 list_train_score.append(train_score)
 net.eval()
for idx, sample in enumerate(val_loader):
data, label = sample
data, label = data.to(device), label.to(device)
 with torch.no_grad() :
out = net(data)
 val_out.append(out.cpu().numpy())
 loss = criterion(label, out)
 score_ = score(label, out)
 val_loss += loss.item() /val_num_batch
val_score += score_.item() /val_num_batch
list_val_loss.append(val_loss)
 list_val_score.append(val_score)
 early_stopping(val_loss, net)
 if early_stopping.early_stop:
print("Early stopping")
 break
scheduler.step(val_loss)
 print(f"train loss: {train_loss} train score: {train_score}
\
val loss: {val_loss} val score: {val_score}
\
final lr: {optimizer.param_groups[0]['lr']}" 
)
best_alpha=1000
best_itr=2700
final_reg_case=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case.fit(X_train_cases,y_train_cases )<predict_on_test>
print("predict test...")
 for k in range(NFOLD):
pred = []
net = make_model(in_ch=len(FE)).to(device)
 net.load_state_dict(torch.load(f"Osic-NN-fold_{k}.pth"))
net.eval()
for idx, sample in enumerate(test_loader):
data = sample
data = data.to(device)
 with torch.no_grad() :
out = net(data)
 out =(out.cpu().numpy() ).astype(np.int)
 pred.append(out)
 pred = np.concatenate(pred)
 pe += pred / NFOLD
pred=final_reg_case.predict(X_test_cases)
 print("The RMSE value",(mean_squared_error(y_test_cases,pred)) **0.5 )<train_on_grid>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = abs(pe[:, 2] - pe[:, 0] )
reg_fatal=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=3800)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_fatal, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_fatal, y_train_fatal)
 results = pd.DataFrame.from_dict(clf.cv_results_ )<train_model>
subm = sub[['Patient_Week','FVC1','Confidence1']].copy()
best_alpha=100
best_iter=3800
final_reg_fatal = ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_iter)
 final_reg_fatal.fit(X_train_fatal, y_train_fatal )<predict_on_test>
subm.rename(columns={'FVC1':'FVC','Confidence1':'Confidence'},inplace=True )
<split><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )