Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<train_model>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
rf = RandomForestRegressor(n_estimators = 1000, random_state = 42)
 rf1 = RandomForestRegressor(n_estimators = 1000, random_state = 42)
 X = pred[['Province_State','Country_Region','Date']]
y = pred['ConfirmedCases']
y2 = pred['Fatalities']
rf.fit(X, y)
 rf1.fit(X,y2)
 trees = tree.DecisionTreeClassifier()
trees1 = tree.DecisionTreeClassifier()
trees.fit(X,y)
 trees1.fit(X,y2 )<feature_engineering>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
for i in range(0,len(test)) :
dt = test['Date'][i]
dt = datetime.strptime(dt, '%Y-%m-%d')
 dt = dt.timestamp() %150000000
dt = dt/100
test['Date'][i] = dt
test['Country_Region'] = le.fit_transform(test['Country_Region'])
 test['Province_State'] = test['Province_State'].replace(np.nan,'aawa')
 test['Province_State'] = le.fit_transform(test['Province_State'])
 x_test = test[['Province_State','Country_Region','Date']]
y_pred = trees.predict(x_test)
 deaths = rf1.predict(x_test)
 row_list = [["ForecastId","ConfirmedCases","Fatalities"]]
for i in range(0,len(test)) :
temp = [test['ForecastId'][i],y_pred[i],deaths[i]]
row_list.append(temp)
 print(row_list)
 with open('submission.csv', 'w', newline='')as file:
writer = csv.writer(file)
 writer.writerows(row_list )<load_from_csv>
warnings.filterwarnings("ignore")
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 train['CRPS']=train.Country_Region+train.Province_State.fillna('')
 test['CRPS']=test.Country_Region+test.Province_State.fillna('')
 train<feature_engineering>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
train['LConfirmedCases']=np.log1p(train['ConfirmedCases'])
 train['LFatalities']=np.log1p(train['Fatalities'])
 train['LDConfirmedCases']=train.groupby('CRPS')[['LConfirmedCases']].diff()
train['LDFatalities']=train.groupby('CRPS')[['LFatalities']].diff()
train['LConfirmedCases1']=train.groupby('CRPS')[['LConfirmedCases']].shift(1)
 train['LConfirmedCases2']=train.groupby('CRPS')[['LConfirmedCases']].shift(2)
 train['LConfirmedCases3']=train.groupby('CRPS')[['LConfirmedCases']].shift(3)
 train['LConfirmedCases4']=train.groupby('CRPS')[['LConfirmedCases']].shift(4)
 train['LDConfirmedCases1']=train.groupby('CRPS')[['LDConfirmedCases']].shift(1)
 train['LDConfirmedCases2']=train.groupby('CRPS')[['LDConfirmedCases']].shift(2)
 train['LDConfirmedCases3']=train.groupby('CRPS')[['LDConfirmedCases']].shift(3)
 train['LDConfirmedCases4']=train.groupby('CRPS')[['LDConfirmedCases']].shift(4)
 train['LDConfirmedCasesMA']=(train.LDConfirmedCases+train.LDConfirmedCases1+train.LDConfirmedCases2+train.LDConfirmedCases3+train.LDConfirmedCases4)/5
train['LDConfirmedCasesMA1']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(1)
 train['LDConfirmedCasesMA2']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(2)
 train['LDConfirmedCasesMA3']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(3)
 train['LDConfirmedCasesMA4']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(4)
 train['LConfirmedCasesMA']=(train.LConfirmedCases+train.LConfirmedCases1+train.LConfirmedCases2+train.LConfirmedCases3+train.LConfirmedCases4)/5
train['LConfirmedCasesMA1']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(1)
 train['LConfirmedCasesMA2']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(2)
 train['LConfirmedCasesMA3']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(3)
 train['LConfirmedCasesMA4']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(4)
 train['LFatalities1']=train.groupby('CRPS')[['LFatalities']].shift(1)
 train['LFatalities2']=train.groupby('CRPS')[['LFatalities']].shift(2)
 train['LFatalities3']=train.groupby('CRPS')[['LFatalities']].shift(3)
 train['LFatalities4']=train.groupby('CRPS')[['LFatalities']].shift(4)
 train['LDFatalities1']=train.groupby('CRPS')[['LDFatalities']].shift(1)
 train['LDFatalities2']=train.groupby('CRPS')[['LDFatalities']].shift(2)
 train['LDFatalities3']=train.groupby('CRPS')[['LDFatalities']].shift(3)
 train['LDFatalities4']=train.groupby('CRPS')[['LDFatalities']].shift(4)
 train['LDFatalitiesMA']=(train.LDFatalities+train.LDFatalities1+train.LDFatalities2+train.LDFatalities3+train.LDFatalities4)/5
train['LDFatalitiesMA1']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(1)
 train['LDFatalitiesMA2']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(2)
 train['LDFatalitiesMA3']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(3)
 train['LDFatalitiesMA4']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(4)
 train['LFatalitiesMA']=(train.LFatalities+train.LFatalities1+train.LFatalities2+train.LFatalities3+train.LFatalities4)/5
train['LFatalitiesMA1']=train.groupby('CRPS')[['LFatalitiesMA']].shift(1)
 train['LFatalitiesMA2']=train.groupby('CRPS')[['LFatalitiesMA']].shift(2)
 train['LFatalitiesMA3']=train.groupby('CRPS')[['LFatalitiesMA']].shift(3)
 train['LFatalitiesMA4']=train.groupby('CRPS')[['LFatalitiesMA']].shift(4)
 train['serd']=train.groupby('CRPS' ).cumcount()
train.loc[train.ConfirmedCases==0,'days_since_confirmed']=0
train.loc[train.ConfirmedCases>0,'days_since_confirmed']=train[train.ConfirmedCases>0].groupby('CRPS' ).cumcount() 
<choose_model_class>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 130
lgbm_cc=LGBMRegressor(num_leaves = 85,learning_rate =10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples =2,subsample =0.97,subsample_freq=10,
colsample_bytree = 0.68,reg_lambda=10**1.4,random_state=1234,n_jobs=4)
 lgbm_f=LGBMRegressor(num_leaves = 26,learning_rate =10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples =14,subsample =0.66,subsample_freq=5,
colsample_bytree = 0.8,reg_lambda=10**1.92,random_state=1234,n_jobs=4)
 oe=OrdinalEncoder()
X=oe.fit_transform(train[['Country_Region','Province_State']].fillna(''))
train['CR']=X[:,0]
train['PS']=X[:,1]
lgbm_cc.fit(train.loc[:,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4']],train.LDConfirmedCases,categorical_feature=['CR','PS'])
 lgbm_f.fit(train.loc[:,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4','LConfirmedCases','LDConfirmedCases']],train.LDFatalities,categorical_feature=['CR','PS'])
<feature_engineering>
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv" )
train['serd']=train.groupby('CRPS' ).cumcount()
trainpred = pd.concat(( train,test[test.Date>train.Date.max() ])).reset_index(drop=True)
 trainpred.sort_values(['Country_Region','Province_State','Date'],inplace=True)
 X=oe.transform(trainpred[['Country_Region','Province_State']].fillna(''))
trainpred['CR']=X[:,0]
trainpred['PS']=X[:,1]
trainpred['serd']=trainpred.groupby('CRPS' ).cumcount()
trainpred.loc[trainpred.ConfirmedCases.isnull() ,'ConfirmedCases']=1
trainpred.loc[trainpred.ConfirmedCases==0,'days_since_confirmed']=0
trainpred.loc[trainpred.ConfirmedCases>0,'days_since_confirmed']=trainpred[trainpred.ConfirmedCases>0].groupby('CRPS' ).cumcount()
trainpred['LConfirmedCases']=np.log1p(trainpred['ConfirmedCases'])
 trainpred['LFatalities']=np.log1p(trainpred['Fatalities'])
 trainpred['LDConfirmedCases']=trainpred.groupby('CRPS')[['LConfirmedCases']].diff()
trainpred['LDFatalities']=trainpred.groupby('CRPS')[['LFatalities']].diff()
for serd in range(train.serd.max() +1,trainpred.serd.max() +1):
print(serd)
 trainpred['LConfirmedCases1']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(1)
 trainpred['LConfirmedCases2']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(2)
 trainpred['LConfirmedCases3']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(3)
 trainpred['LConfirmedCases4']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(4)
 trainpred['LDConfirmedCases1']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(1)
 trainpred['LDConfirmedCases2']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(2)
 trainpred['LDConfirmedCases3']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(3)
 trainpred['LDConfirmedCases4']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(4)
 trainpred['LDConfirmedCasesMA']=(trainpred.LDConfirmedCases+trainpred.LDConfirmedCases1+trainpred.LDConfirmedCases2+trainpred.LDConfirmedCases3+trainpred.LDConfirmedCases4)/5
trainpred['LDConfirmedCasesMA1']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(1)
 trainpred['LDConfirmedCasesMA2']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(2)
 trainpred['LDConfirmedCasesMA3']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(3)
 trainpred['LDConfirmedCasesMA4']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(4)
 trainpred['LConfirmedCasesMA']=(trainpred.LConfirmedCases+trainpred.LConfirmedCases1+trainpred.LConfirmedCases2+trainpred.LConfirmedCases3+trainpred.LConfirmedCases4)/5
trainpred['LConfirmedCasesMA1']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(1)
 trainpred['LConfirmedCasesMA2']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(2)
 trainpred['LConfirmedCasesMA3']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(3)
 trainpred['LConfirmedCasesMA4']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(4)
 trainpred['LFatalities1']=trainpred.groupby('CRPS')[['LFatalities']].shift(1)
 trainpred['LFatalities2']=trainpred.groupby('CRPS')[['LFatalities']].shift(2)
 trainpred['LFatalities3']=trainpred.groupby('CRPS')[['LFatalities']].shift(3)
 trainpred['LFatalities4']=trainpred.groupby('CRPS')[['LFatalities']].shift(4)
 trainpred['LDFatalities1']=trainpred.groupby('CRPS')[['LDFatalities']].shift(1)
 trainpred['LDFatalities2']=trainpred.groupby('CRPS')[['LDFatalities']].shift(2)
 trainpred['LDFatalities3']=trainpred.groupby('CRPS')[['LDFatalities']].shift(3)
 trainpred['LDFatalities4']=trainpred.groupby('CRPS')[['LDFatalities']].shift(4)
 trainpred['LDFatalitiesMA']=(trainpred.LDFatalities+trainpred.LDFatalities1+trainpred.LDFatalities2+trainpred.LDFatalities3+trainpred.LDFatalities4)/5
trainpred['LDFatalitiesMA1']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(1)
 trainpred['LDFatalitiesMA2']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(2)
 trainpred['LDFatalitiesMA3']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(3)
 trainpred['LDFatalitiesMA4']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(4)
 trainpred['LFatalitiesMA']=(trainpred.LFatalities+trainpred.LFatalities1+trainpred.LFatalities2+trainpred.LFatalities3+trainpred.LFatalities4)/5
trainpred['LFatalitiesMA1']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(1)
 trainpred['LFatalitiesMA2']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(2)
 trainpred['LFatalitiesMA3']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(3)
 trainpred['LFatalitiesMA4']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(4)
 trainpred.loc[trainpred.serd==serd,'LDConfirmedCases']= lgbm_cc.predict(trainpred.loc[trainpred.serd==serd,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4']])
 trainpred.loc[(trainpred.serd==serd)&(trainpred.LDConfirmedCases<0),'LDConfirmedCases']=0
trainpred.loc[trainpred.serd==serd,'LConfirmedCases']=trainpred.loc[trainpred.serd==serd,'LDConfirmedCases']+trainpred.loc[trainpred.serd==serd,'LConfirmedCases1']
trainpred.loc[trainpred.serd==serd,'ConfirmedCases']=np.exp(trainpred.loc[trainpred.serd==serd,'LConfirmedCases'])-1
trainpred.loc[trainpred.serd==serd,'LDFatalities']= lgbm_f.predict(trainpred.loc[trainpred.serd==serd,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4','LConfirmedCases','LDConfirmedCases']])
 trainpred.loc[(trainpred.serd==serd)&(trainpred.LDFatalities<0),'LDFatalities']=0
trainpred.loc[trainpred.serd==serd,'LFatalities']=trainpred.loc[trainpred.serd==serd,'LDFatalities']+trainpred.loc[trainpred.serd==serd,'LFatalities1']
trainpred.loc[trainpred.serd==serd,'Fatalities']=np.exp(trainpred.loc[trainpred.serd==serd,'LFatalities'])-1<save_to_csv>
sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )
trainpred.loc[(trainpred.Date<=max(test.Date)) &(trainpred.Date>=min(test.Date)) ,'ForecastId']=test.loc[:,'ForecastId'].values
submission=trainpred.loc[trainpred.Date>=min(test.Date)][['ForecastId','ConfirmedCases','Fatalities']]
submission.ForecastId=submission.ForecastId.astype('int')
 submission.sort_values('ForecastId',inplace=True)
 submission.to_csv('submission.csv',index=False )<import_modules>
data = train.append([test, sub] )
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
from sklearn.preprocessing import LabelBinarizer,LabelEncoder,StandardScaler,MinMaxScaler
from sklearn.metrics import mean_squared_log_error 
<load_from_csv>
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub] )
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )<count_missing_values>
print('Training Data:',train.info() , end = "
")
 print('Testing Data:',test.info() )
train_df.isna().sum()<count_missing_values>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
test_df.isna().sum()<categorify>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
train_df['Province_State'].fillna('',inplace=True)
 test_df['Province_State'].fillna('',inplace=True)
 lb = LabelEncoder()
train_df['Country_Region'] = lb.fit_transform(train_df['Country_Region'])
 test_df['Country_Region'] = lb.transform(test_df['Country_Region'])
 lb1 = LabelEncoder()
train_df['Province_State'] = lb.fit_transform(train_df['Province_State'])
 test_df['Province_State'] = lb.transform(test_df['Province_State'])
 def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
train_df.Date = train_df.Date.apply(split_date)
 test_df.Date = test_df.Date.apply(split_date)
 year = []
month = []
day = []
for i in train_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 train_df['Year'] = year
train_df['Month'] = month
train_df['Day'] = day
del train_df['Date']
year = []
month = []
day = []
for i in test_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 test_df['Year'] = year
test_df['Month'] = month
test_df['Day'] = day
del test_df['Date']
del train_df['Id']
del test_df['ForecastId']
del train_df['Year']
del test_df['Year']
train_df['ConfirmedCases'] = train_df['ConfirmedCases'].apply(int)
 train_df['Fatalities'] = train_df['Fatalities'].apply(int)
 cases = train_df.ConfirmedCases
fatalities = train_df.Fatalities
del train_df['ConfirmedCases']
del train_df['Fatalities']
scaler = MinMaxScaler()
X = scaler.fit_transform(train_df.values)
 x_test = scaler.transform(test_df.values)
<split>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
<train_model>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
xg = XGBRegressor(n_estimators = 10*1000 , random_state = 0 , max_depth = 15)
 xg.fit(X,cases)
 cases_pred = xg.predict(x_test)
 cases_pred[cases_pred < 0] = 0 
<import_modules>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
<split>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
X_cas = []
for i in range(len(X)) :
x = list(X[i])
 x.append(cases[i])
 X_cas.append(x)
 X_cas = np.array(X_cas)
<train_model>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
rf = XGBRegressor(n_estimators = 10000 , random_state = 0 , max_depth = 15)
 rf.fit(X,fatalities)
 fatalities_pred = rf.predict(x_test)
 fatalities_pred[fatalities_pred < 0] = 0 
<prepare_output>
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
submission_df['ConfirmedCases'] = cases_pred
submission_df['Fatalities'] = fatalities_pred<save_to_csv>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
submission_df.to_csv("submission.csv" , index = False )<set_options>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=2000,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )<load_from_disk>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
train_data = pd.read_json('.. /input/train.json')
 test_data = pd.read_json('.. /input/test.json' )<train_model>
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
print("The training data consists of {} recipes".format(len(train_data)) )<count_unique_values>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
print("Number of cuisine categories: {}".format(len(train_data.cuisine.unique())))
 train_data.cuisine.unique()<define_variables>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
def random_colours(number_of_colors):

 colors = []
for i in range(number_of_colors):
colors.append("
return colors<count_values>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<filter><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )