Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<choose_model_class>
import numpy as np
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns
class TRMFRegressor(BaseEstimator):
def __init__(self,
n_components,
n_order,
C_Z=1e-1,
C_F=1e-1,
C_phi=1e-2,
eta_Z=0.5,
eta_F=0.,
adj=None,
C_B=0.0,
fit_regression=False,
fit_intercept=True,
nonnegative_factors=True,
tol=1e-5,
n_max_iterations=1000,
n_max_mf_iter=5,
z_step_kind="tron",
f_step_kind="tron",
random_state=None):
super(TRMFRegressor, self ).__init__()
self.n_components = n_components
self.n_order = n_order
self.C_Z = C_Z
self.C_F = C_F
self.C_phi = C_phi
self.eta_Z = eta_Z
self.eta_F = eta_F
self.adj = adj
self.C_B = C_B
self.fit_regression = fit_regression
self.fit_intercept = fit_intercept
self.tol = tol
self.n_max_iterations = n_max_iterations
self.n_max_mf_iter = n_max_mf_iter
self.nonnegative_factors = nonnegative_factors
self.random_state = random_state
self.z_step_kind = z_step_kind
self.f_step_kind = f_step_kind
def fit(self, X, y=None, sample_weight=None):
if not self.fit_regression:
if y is not None:
raise TypeError(
)
 X, y = None, X
else:
if y is None:
raise TypeError(
)
 if X is None:
X = np.empty(( y.shape[0], 0))
check_consistent_length(X, y)
 f_step_kind = "fgm" if self.nonnegative_factors else self.f_step_kind
estimates = trmf(y, self.n_components, self.n_order, self.C_Z,
self.C_F, self.C_phi, self.eta_Z, self.eta_F,
adj=self.adj, fit_intercept=self.fit_intercept,
regressors=X, C_B=self.C_B, tol=self.tol,
n_max_iterations=self.n_max_iterations,
n_max_mf_iter=self.n_max_mf_iter,
f_step_kind=f_step_kind,
z_step_kind=self.z_step_kind,
random_state=self.random_state)
 factors, loadings, ar_coef, intercept, beta = estimates
self.factors_, self.loadings_ = factors, loadings
self.ar_coef_ = ar_coef
self.coef_, self.intercept_ = beta, intercept
return self
def fit_predict(self, n_ahead):
fitted_targets = np.dot(self.factors_,self.loadings_)
 forecast_targets = trmf_forecast_targets(n_ahead, self.loadings_, self.ar_coef_, self.intercept_, self.coef_,self.factors_)
 predicted = np.concatenate([fitted_targets,forecast_targets],axis=0)
 return predicted
def forecast_factors(self, n_ahead):
return trmf_forecast_factors(n_ahead, self.ar_coef_,
prehist=self.factors_)
 def predict(self, X=None, n_ahead=10):
if self.fit_regression:
X = check_array(X, dtype="numeric", accept_sparse=False)
 else:
X = np.empty(( n_ahead, 0))
return trmf_forecast_targets(
 n_ahead, self.loadings_, self.ar_coef_, self.intercept_,
self.coef_, self.factors_ )<load_from_csv>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000 )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<data_type_conversions>
df_train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )
train.rename(columns={'Country_Region':'Country'}, inplace=True)
 train.rename(columns={'Province_State':'State'}, inplace=True)
 train.State=train.State.fillna('NA')
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 test.rename(columns={'Country_Region':'Country'}, inplace=True)
 test.rename(columns={'Province_State':'State'}, inplace=True)
 test.State=test.State.fillna('NA')
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True )<categorify>
print(f'Training Set Shape = {df_train.shape} - Patients = {df_train["Patient"].nunique() }')
 print(f'Training Set Memory Usage = {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')
 print(f'Test Set Shape = {df_test.shape} - Patients = {df_test["Patient"].nunique() }')
 print(f'Test Set Memory Usage = {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB' )
le_state_country = preprocessing.LabelEncoder()
train['state_country_id'] = le_state_country.fit_transform(train.State.astype('str')+":"+train.Country.astype('str'))
test['state_country_id'] = le_state_country.transform(test.State.astype('str')+":"+test.Country.astype('str'))<rename_columns>
training_sample_counts = df_train.rename(columns={'Weeks': 'Samples'} ).groupby('Patient' ).agg('count')['Samples'].value_counts()
print(f'Training Set FVC Measurements Per Patient
{("-")* 41}
{training_sample_counts}' )
train_confirmed_cases = train[['state_country_id','Date','ConfirmedCases']]
train_confirmed_cases.rename(columns={'ConfirmedCases':'y'},inplace=True)
 train_fatalities = train[['state_country_id','Date','Fatalities']]
train_fatalities.rename(columns={'Fatalities':'y'},inplace=True )<drop_column>
df_submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 df_submission.head()
train_fatalities = train[['state_country_id','Date','Fatalities','ConfirmedCases']]
train_fatalities.drop(columns=['ConfirmedCases'],inplace=True)
 train_fatalities.rename(columns={'Fatalities':'y'},inplace=True )<normalization>
warnings.filterwarnings("ignore" )
def normalise(df, scaler=MinMaxScaler()):
df['y_norm'] = scaler.fit_transform(df['y'].values.reshape(len(df), 1)).reshape(len(df))
return df
def inverse_normalise(s_c_min_max, y, s_c):
return y *(s_c_min_max[s_c]['max'] - s_c_min_max[s_c]['min'])+ s_c_min_max[s_c]['min']
def preprocess(df,s_c_ids,dates):
working_df = df
df_normalised_y = []
s_c_min_max = {}
for i, considering_s_c in enumerate(s_c_ids):
source_df = working_df[working_df['state_country_id'] == considering_s_c]
s_c_min_max[considering_s_c] = {'min': source_df['y'].min() , 'max': source_df['y'].max() }
for d in dates:
if d not in(source_df.Date.unique()):
source_df = source_df.append({'state_country_id':considering_s_c,'Date':d, 'y':0} , ignore_index=True)
 source_df = normalise(source_df)
 df_normalised_y.append(source_df)
 df_normalised_y = pd.concat(df_normalised_y ).reset_index(drop=True)
 working_df = df_normalised_y
working_df = working_df.sort_values(by=['state_country_id','Date'] ).reset_index(drop=True)
 table = pd.pivot_table(working_df[['state_country_id','Date','y_norm']], values ='y_norm', index =['Date'],
columns =['state_country_id'])
 df_input = pd.DataFrame(table.to_records() ).set_index('Date')
 targets = df_input.values
return targets,s_c_min_max
def to_original_format(data,s_c_ids,dates,s_c_min_max):
d = pd.DataFrame(data,columns = s_c_ids,index = dates)
 d = pd.melt(d.rename_axis('Date' ).reset_index() , var_name='state_country_id', value_name='y_norm', id_vars='Date')
 d = d.sort_values(by=['state_country_id','Date'])
 result = []
for i, s_c_id in enumerate(s_c_ids):
temp = d[d['state_country_id']==s_c_id]
temp['y'] = temp['y_norm'].apply(lambda y: inverse_normalise(s_c_min_max,y,s_c_id))
result.append(temp)
 result = pd.concat(result ).reset_index(drop=True)
 return result
def predict(targets,params,n_ahead,mode='training'):
if mode=='training':
train_targets, test_targets = train_test_split(targets, test_size=11, shuffle=False)
 n_ahead = len(test_targets)
 else:
train_targets = targets
print("Training the model...")
 regressor = TRMFRegressor(**params)
 regressor.fit(train_targets)
 print("Forecasting...")
 predicted = regressor.fit_predict(n_ahead = n_ahead)
 return predicted,n_ahead
def get_forecasts_df(predicted, s_c_ids, dates, n_ahead, s_c_min_max, mode="training"):
if mode!='training':
dates_new = pd.date_range(datetime.strptime(dates[-1:][0], '%Y-%m-%d'),periods=n_ahead+1, freq='1D')[1:]
dates = dates.tolist()
dates.extend(dates_new.strftime("%Y-%m-%d"))
dates = np.array(dates)
 df_pred = to_original_format(predicted, s_c_ids, dates, s_c_min_max)[['state_country_id','Date','y']]
df_pred = df_pred.rename(columns = {'y':'yhat'})
 return df_pred
def cluster_forecasts(active_df,clusterId,s_c_ids,curr_dates,n_ahead,n_components,n_order,mode='training',plot=False):
print(f"Working on cluster - {clusterId}")
 print("setting the parameters...")
 params = {
"n_components" : n_components,
"n_order" : n_order,
"C_Z" : 5e1,
"C_F" : 5e-4,
"C_phi" : 1e-4,
"eta_Z" : 0.25,
"C_B" : 0.,
"fit_regression" : False,
"fit_intercept" : False,
"nonnegative_factors" : False,
"n_max_iterations" : 1000,
"f_step_kind" : "tron",
"tol" : 1e-5,
}
print("Preparing the data...")
 considering_s_cs = np.sort(s_c_ids)
 working_df = active_df[active_df['state_country_id'].isin(considering_s_cs)]
working_df = working_df.sort_values(by=['state_country_id','Date'])[['state_country_id','Date','y']]
targets, s_c_min_max = preprocess(working_df,considering_s_cs,curr_dates)
 predicted,n_ahead = predict(targets,params,n_ahead,mode=mode)
 if plot==True:
print("plotting graphs-------")
 plot_predictions(targets,predicted,len(considering_s_cs),n_ahead,n_cols=4)
 df_pred = get_forecasts_df(predicted, s_c_ids, dates,n_ahead, s_c_min_max,mode=mode)
 df_comb = pd.merge(df_pred,working_df,on=['state_country_id','Date'],how='left')
 return df_comb
def plot_predictions(targets,predicted,no_uuids,n_ahead,n_cols):
n_rows =(no_uuids + n_cols - 1)// n_cols
fig, axes = plt.subplots(n_rows, n_cols, figsize=(12, 1.5*n_rows),
sharex=True, sharey=False)
 for j, ax in zip(range(no_uuids), axes.flat):
ax.plot(targets[:, j], lw=2)
 ax.plot(predicted[:-n_ahead, j], zorder=-1)
 ax.plot(predicted[:, j], zorder=-2, alpha=0.5)
 plt.show()
plt.close()<define_variables>
def seed_everything(seed=777):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
n_ahead = 60<data_type_conversions>
OUTPUT_DICT = './'
ID = 'Patient_Week'
TARGET = 'FVC'
SEED = 777
seed_everything(seed=SEED)
 N_FOLD = 7
train_confirmed_cases['Date'] = pd.to_datetime(train_confirmed_cases['Date'], format='%Y-%m-%d')
 train_confirmed_cases['Date'] = train_confirmed_cases['Date'].dt.strftime('%Y-%m-%d')
 train_confirmed_cases['Date'] = train_confirmed_cases['Date'].apply(lambda row: datetime.strptime(row, "%Y-%m-%d" ).strftime("%Y-%m-%d"))<data_type_conversions>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )
train_fatalities['Date'] = pd.to_datetime(train_confirmed_cases['Date'], format='%Y-%m-%d')
 train_fatalities['Date'] = train_fatalities['Date'].dt.strftime('%Y-%m-%d')
 train_fatalities['Date'] = train_fatalities['Date'].apply(lambda row: datetime.strptime(row, "%Y-%m-%d" ).strftime("%Y-%m-%d"))<sort_values>
train = pd.concat([train,otest])
 output = pd.DataFrame()
gb = train.groupby('Patient')
 tk0 = tqdm(gb, total=len(gb))
for _, usr_df in tk0:
usr_output = pd.DataFrame()
for week, tmp in usr_df.groupby('Weeks'):
rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}
tmp = tmp.rename(columns=rename_cols)
 drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']
_usr_output = usr_df.drop(columns=drop_cols ).rename(columns={'Weeks': 'predict_Week'} ).merge(tmp, on='Patient')
 _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']
usr_output = pd.concat([usr_output, _usr_output])
 output = pd.concat([output, usr_output])
 train = output[output['Week_passed']!=0].reset_index(drop=True )
train_confirmed_cases = train_confirmed_cases[train_confirmed_cases.Date<='2020-04-03'].reset_index(drop=True)
 train_confirmed_cases = train_confirmed_cases.sort_values(by=['state_country_id','Date'] ).reset_index(drop=True)
 train_fatalities = train_fatalities[train_fatalities.Date<='2020-04-03'].reset_index(drop=True)
 train_fatalities = train_fatalities.sort_values(by=['state_country_id','Date'] ).reset_index(drop=True )<feature_engineering>
test = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})
 submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])
 submission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 test = submission.drop(columns=['FVC', 'Confidence'] ).merge(test, on='Patient')
 test['Week_passed'] = test['predict_Week'] - test['base_Week']
test.set_index('Patient_Week', inplace=True )
train_confirmed_cases['y'] = train_confirmed_cases['y'].fillna(0)
 train_fatalities['y'] = train_fatalities['y'].fillna(0)
 dates = np.sort(train_confirmed_cases.Date.unique())
 s_c_ids = np.sort(train_confirmed_cases.state_country_id.unique() )<merge>
folds = train[['Patient', TARGET]].copy()
Fold = GroupKFold(n_splits=N_FOLD)
 groups = folds['Patient'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[TARGET], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int )
fitted_confirmed_cases.rename(columns={'y':'y_cc','yhat':'yhat_cc'},inplace=True)
 fitted_fatalities.rename(columns={'y':'y_f','yhat':'yhat_f'},inplace=True)
 fitted_total = pd.merge(fitted_confirmed_cases,fitted_fatalities,on=['state_country_id','Date'],how='inner' )<compute_test_metric>
def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):
trn_idx = folds[folds.fold!=fold_num].index
val_idx = folds[folds.fold==fold_num].index
y_tr = target.iloc[trn_idx].values
X_tr = train_df.iloc[trn_idx][features].values
y_val = target.iloc[val_idx].values
X_val = train_df.iloc[val_idx][features].values
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
clf.fit(X_tr, y_tr)
 oof[val_idx] = clf.predict(X_val)
 predictions += clf.predict(test_df[features])
 return oof, predictions
def run_kfold_model(clf, train, test, folds, features, target, n_fold=7):
oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_ in range(n_fold):
_oof, _predictions = run_single_model(clf,
train,
test,
folds,
features,
target,
fold_num=fold_)
 oof += _oof
predictions += _predictions/n_fold
return oof, predictions
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 dates_sub = dates[-11:]
sub = fitted_total[fitted_total.Date.isin(dates_sub)].reset_index(drop=True)
 rmsle_confirmed = RMSLE(sub.yhat_cc,sub.y_cc)
 rmsle_fatalites = RMSLE(sub.yhat_f,sub.y_f)
 print(rmsle_confirmed)
 print(rmsle_fatalites)
 print(( rmsle_confirmed+rmsle_fatalites)/2 )<data_type_conversions>
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']
features = [c for c in features if c not in drop_features]
if cat_features:
ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')
 ce_oe.fit(train)
 train = ce_oe.transform(train)
 test = ce_oe.transform(test )
fitted_total['Date'] = pd.to_datetime(fitted_total['Date'], format='%Y-%m-%d')
 fitted_total.head()<data_type_conversions>
for alpha1 in [0.3]:
for l1s in [0.8]:
print(" For alpha:",alpha1,"& l1_ratio:",l1s)
 clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)
 oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)
 train['FVC_pred'] = oof
test['FVC_pred'] = predictions
train['Confidence'] = 100
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score)
 def loss_func(weight, row):
confidence = weight
sigma_clipped = max(confidence, 70)
 diff = abs(row['FVC'] - row['FVC_pred'])
 delta = min(diff, 1000)
 score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)
 return -score
results = []
tk0 = tqdm(train.iterrows() , total=len(train))
for _, row in tk0:
loss_partial = partial(loss_func, row=row)
 weight = [100]
result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')
 x = result['x']
results.append(x[0])
 train['Confidence'] = results
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )
fitted_confirmed_cases['yhat_exp'] = fitted_confirmed_cases['yhat'].astype(int)
 fitted_fatalities['yhat_exp'] = fitted_fatalities['yhat'].astype(int)
<feature_engineering>
TARGET = 'Confidence'
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']
features = [c for c in features if c not in drop_features]
oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD )
test['Date'] = test['Date'].dt.strftime('%Y-%m-%d' )<feature_engineering>
train['Confidence'] = oof
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )
<merge>
test['Confidence'] = predictions
test = test.reset_index()
test_updated = pd.merge(test,pred_confirmed_cases,on=['state_country_id','Date'],how='left')
 test_updated.rename(columns={'yhat':'ConfirmedCases'},inplace=True)
 test_updated.drop(columns=['y'],inplace=True)
 test_updated = pd.merge(test_updated,pred_fatalities,on=['state_country_id','Date'],how='left')
 test_updated.rename(columns={'yhat':'Fatalities'},inplace=True)
 test_updated.drop(columns=['y'],inplace=True )<drop_column>
sub = submission[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')
 sub = sub.rename(columns={'FVC_pred': 'FVC'})
 for i in range(len(otest)) :
sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
sub[sub.Confidence<1]
sub.to_csv('submission_2.csv', index=False, float_format='%.1f' )
test_updated = test_updated[['ForecastId','ConfirmedCases','Fatalities']]
test_updated.head()<save_to_csv>
import numpy as np
import pandas as pd
import pydicom
import os
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
test_updated.to_csv("submission.csv",index=False )<load_from_csv>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
DESIRED_SIZE = 128
train_df = pd.read_csv(PREFIX+"train.csv", parse_dates=["Date"])
 test_df = pd.read_csv(PREFIX+"test.csv", parse_dates=["Date"])
 sub_df = pd.read_csv(PREFIX+"submission.csv")
 LAG_DAYS = 28
MIN_DATE_TRAIN = train_df["Date"].min()
MID_DATE_TRAIN = train_df["Date"].min() + timedelta(days=LAG_DAYS)
 MAX_DATE_TRAIN = train_df["Date"].max()
START_DATE_TRAIN = MID_DATE_TRAIN + timedelta(days=1)
 MIN_DATE_TEST = test_df["Date"].min()
MAX_DATE_TEST = test_df["Date"].max()
print(MIN_DATE_TRAIN)
 print(MID_DATE_TRAIN)
 print(MAX_DATE_TRAIN)
 print(MIN_DATE_TEST)
 print(MAX_DATE_TEST)
 print('done')
 both_df = pd.concat([train_df,test_df[test_df["Date"]>MAX_DATE_TRAIN]],axis=0)
 print(train_df.shape)
 print(test_df.shape)
 print(both_df.shape)
 both_df.head(3)
 oof_df = train_df[train_df["Date"]<MID_DATE_TRAIN]
def create_oof_dict(oof_df):
first_death_list = []
first_case_list = []
country_list = oof_df["Country_Region"].unique().tolist()
for country in country_list:
df = oof_df[oof_df['Country_Region']==country]
df1 = df[df["ConfirmedCases"]>0]
if df1.shape[0] > 0:
first_case_list.append(df1["Date"].min())
 else:
first_case_list.append(MAX_DATE_TEST)
 df2 = df[df["Fatalities"]>0]
if df2.shape[0] > 0:
first_death_list.append(df2["Date"].min())
 else:
first_death_list.append(MAX_DATE_TEST)
 first_df = pd.DataFrame()
first_df['country'] = country_list
first_df['first_death_date'] = first_death_list
first_df['first_case_date'] = first_case_list
first_df['first_death_date'] =(first_df['first_death_date']-MIN_DATE_TRAIN ).dt.days
first_df['first_case_date'] =(first_df['first_case_date']-MIN_DATE_TRAIN ).dt.days
death_map = dict(zip(first_df['country'].tolist() ,first_df['first_death_date'].tolist()))
case_map = dict(zip(first_df['country'].tolist() ,first_df['first_case_date'].tolist()))
return case_map, death_map
case_map, death_map = create_oof_dict(oof_df)
 print("done")
 def replace_state_nan(df):
df = df.copy()
bool_index = df['Province_State'].isnull()
df.loc[bool_index, 'Province_State'] = df.loc[bool_index,'Country_Region'] + "_NaN"
return df
def create_days(df):
df = df.copy()
df['Days'] =(df["Date"]-MIN_DATE_TRAIN ).dt.days.astype(int)
 df['weekday'] = df["Date"].dt.dayofweek
return df
def create_CFR(df):
df = df.copy()
df["CFR"] = df["Fatalities"]/df["ConfirmedCases"]
df["CFR"].fillna(0, inplace=True)
 return df
korea_dummy_dict = defaultdict(int)
 korea_dummy_dict['Korea, South'] = 1
hubei_dummy_dict = defaultdict(int)
 hubei_dummy_dict['Hubei'] = 1
iran_dummy_dict = defaultdict(int)
 iran_dummy_dict['Iran'] = 1
us_dummy_dict = defaultdict(int)
 us_dummy_dict['US'] = 1
italy_dummy_dict = defaultdict(int)
 italy_dummy_dict['Italy'] = 1
COUNTRY_VC = train_df["Country_Region"].value_counts()
def hash_country(df):
df = df.copy()
df['country_hash'] = pd.util.hash_array(df['Country_Region'])
 df['country_hash'] = df['country_hash'].rank(pct=True)
 df['country_count'] = df['Country_Region'].map(COUNTRY_VC)
 df['state_hash'] = pd.util.hash_array(df['Province_State'])
 df['state_hash'] = df['Province_State'].rank(pct=True)
 df['first_death'] = df["Country_Region"].map(death_map)
 df['first_case'] = df["Country_Region"].map(case_map)
 df['dummy_korea'] = df["Country_Region"].map(korea_dummy_dict)
 df['dummy_hubei'] = df["Province_State"].map(hubei_dummy_dict)
 df['dummy_iran'] = df["Country_Region"].map(iran_dummy_dict)
 df['dummy_italy'] = df["Country_Region"].map(italy_dummy_dict)
 return df
def create_features(df):
df = replace_state_nan(df)
 df = hash_country(df)
 df = create_days(df)
 return df
both_df = create_features(both_df)
 both_df['state_date_key'] = both_df["Province_State"].astype(str)+ "_" + both_df["Date"].astype(str)
 case_dict = dict(zip(both_df['state_date_key'].tolist() , both_df['ConfirmedCases'].tolist()))
fatal_dict = dict(zip(both_df['state_date_key'].tolist() , both_df['Fatalities'].tolist()))
both_df.head(3)
 target_to_features_dict = {}
STATIC_FEATURES_LIST = ["country_hash","country_count","state_hash",
"first_death",
"first_case",
"Days","weekday",
"dummy_korea","dummy_hubei","dummy_iran"]
province_list = both_df["Province_State"].unique().tolist()
date_array = both_df[(both_df["Date"]>START_DATE_TRAIN)&(both_df["Date"]<=MAX_DATE_TRAIN)]["Date"].unique()
both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 def update_target_dict(both_df, current_date, current_province_state):
features_list = []
target_df = both_df[(both_df["Province_State"]==current_province_state)&(both_df["Date"] == current_date)]
cases = target_df["ConfirmedCases"].values[0]
fatals = target_df["Fatalities"].values[0]
current_key = target_df["state_date_key"].values[0]
prior_df = both_df[(both_df["Province_State"]==current_province_state)& 
(both_df["Date"] < current_date)].copy()
prior_df.sort_values("Date",inplace=True)
 prior_df = prior_df.tail(LAG_DAYS ).copy()
prior_df.reset_index(drop=True, inplace=True)
 features_list += [prior_df["Fatalities"].max() ]
features_list += [prior_df["ConfirmedCases"].max() ]
features_list += [prior_df["CFR"].max() ]
features_list += [(prior_df["Fatalities"]*prior_df["ConfirmedCases"] ).max() ]
features_list += [prior_df["Fatalities"].diff().abs().max() ]
features_list += [prior_df["ConfirmedCases"].diff().abs().max() ]
features_list += [prior_df["CFR"].diff().abs().max() ]
features_list += target_df[STATIC_FEATURES_LIST].values.tolist() [0]
for f in ["ConfirmedCases","Fatalities","CFR"]:
features_list += prior_df[f].tolist()
for f in ["ConfirmedCases","Fatalities"]:
features_list +=(prior_df[f].diff(2)/(1+prior_df[f])).dropna().tolist()
features_list +=(-prior_df["ConfirmedCases"]+prior_df["Fatalities"] ).tolist()
model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].shift(periods=2 ).dropna() ,
prior_df[["ConfirmedCases"]].shift(periods=-2 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]])
 features_list += [preds[-1]]
model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].shift(periods=2 ).dropna() ,
prior_df[["Fatalities"]].shift(periods=-2 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]])
 features_list += [preds[-1]]
model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(periods=1 ).dropna() ,
prior_df[["ConfirmedCases"]].diff(periods=-1 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(1 ).dropna())
 features_list += [preds[-1]]
model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(periods=1 ).dropna() ,
prior_df[["Fatalities"]].diff(periods=-1 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(1 ).dropna())
 features_list += [preds[-1]]
features_list += [(prior_df["Days"]-prior_df["first_death"] ).max() ]
features_list += [(prior_df["Days"]-prior_df["first_case"] ).max() ]
target_to_features_dict[(current_province_state, current_date, current_key)] = [cases,fatals,features_list]
for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 def list_to_matrix(x):
return np.array([np.array(xi)for xi in x])
 def create_train_X_y(target_to_features_dict):
X_train = []
y_key = []
y_train_case = []
y_train_fatal = []
for k,v in target_to_features_dict.items() :
X_train.append(v[2])
 y_train_case.append(v[0])
 y_train_fatal.append(v[1])
 y_key.append(k[2])
 X_train = list_to_matrix(X_train)
 return X_train, y_train_case, y_train_fatal, y_key
X_train, y_train_case, y_train_fatal, y_key = create_train_X_y(target_to_features_dict)
 print(X_train.shape)
 print("done")
 def create_test_X_y(target_to_features_dict, date_array_0):
X_test = []
y_key = []
y_test_case = []
y_test_fatal = []
for k,v in target_to_features_dict.items() :
if k[1] == date_array_0:
X_test.append(v[2])
 y_test_case.append(v[0])
 y_test_fatal.append(v[1])
 y_key.append(k[2])
 X_test = list_to_matrix(X_test)
 return X_test, y_test_case, y_test_fatal, y_key
np.random.seed(2)
 def rmsle(y_true, y_pred):
return 'RMSLE', np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2))), False
num_features = X_train.shape[1]
mc = ["0"]*(num_features-4)
 MC_STR = "(1,1,1,1," + ",".join(mc)+ ")"
def xgb_reg() :
return XGBRegressor(n_estimators=2000,
booster="gbtree",
tree_method = "exact",
eta= 0.003,
max_depth=6,
subsample=0.9,
colsample_bytree=0.9,
colsample_bylevel=0.9,
min_child_weight = 0,
objective = "reg:squarederror",
reg_alpha=0.1,
reg_lambda=0.95)
 def dual_xgb_fit(X_train, y_train_case, y_train_fatal):
gbm_case = xgb_reg()
gbm_case.fit(X_train, np.log1p(y_train_case))
gbm_fatal = xgb_reg()
gbm_fatal.fit(X_train, np.log1p(y_train_fatal))
return gbm_case, gbm_fatal
gbm_case, gbm_fatal = dual_xgb_fit(X_train, y_train_case, y_train_fatal)
 print("done")
 print(both_df["state_date_key"].nunique())
 print(both_df.shape)
 print(both_df["Date"][both_df["Fatalities"].isnull() ].min())
 date_array = [both_df["Date"][both_df["Fatalities"].isnull() ].min() ]
print(date_array)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 X_test, y_test_case, y_test_fatal, y_key = create_test_X_y(target_to_features_dict, date_array[0])
 print(X_test.shape)
 y_pred_case = np.expm1(gbm_case.predict(X_test))
y_pred_fatal = np.expm1(gbm_fatal.predict(X_test))
print("done")
 for j in range(len(y_key)) :
key_val = y_key[j]
case_dict[key_val] = y_pred_case[j]
fatal_dict[key_val] = y_pred_fatal[j]
print("done")
 both_df["ConfirmedCases"] = both_df["state_date_key"].map(case_dict)
 both_df["Fatalities"] = both_df["state_date_key"].map(fatal_dict)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 print("done")
 print(date_array)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 X_train, y_train_case, y_train_fatal, y_key = create_train_X_y(target_to_features_dict)
 print(X_train.shape)
 print("done")
 gbm_case, gbm_fatal = dual_xgb_fit(X_train, y_train_case, y_train_fatal)
 print("done")
 date_array = [both_df["Date"][both_df["Fatalities"].isnull() ].min() ]
print(date_array)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 X_test, y_test_case, y_test_fatal, y_key = create_test_X_y(target_to_features_dict, date_array[0])
 print(X_test.shape)
 y_pred_case = np.expm1(gbm_case.predict(X_test))
y_pred_fatal = np.expm1(gbm_fatal.predict(X_test))
print("done")
 for j in range(len(y_key)) :
key_val = y_key[j]
case_dict[key_val] = y_pred_case[j]
fatal_dict[key_val] = y_pred_fatal[j]
print(len(case_dict.keys()))
print("done")
 both_df["ConfirmedCases"] = both_df["state_date_key"].map(case_dict)
 both_df["Fatalities"] = both_df["state_date_key"].map(fatal_dict)
 print("done")
 null_count = both_df["Fatalities"].isnull().sum()
print(null_count)
 while null_count > 0:
print(null_count)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 X_train, y_train_case, y_train_fatal, y_key = create_train_X_y(target_to_features_dict)
 gbm_case, gbm_fatal = dual_xgb_fit(X_train, y_train_case, y_train_fatal)
 date_array = [both_df["Date"][both_df["Fatalities"].isnull() ].min() ]
print(date_array)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 X_test, y_test_case, y_test_fatal, y_key = create_test_X_y(target_to_features_dict, date_array[0])
 print(X_test.shape)
 y_pred_case = np.expm1(gbm_case.predict(X_test))
y_pred_fatal = np.expm1(gbm_fatal.predict(X_test))
for j in range(len(y_key)) :
key_val = y_key[j]
case_dict[key_val] = y_pred_case[j]
fatal_dict[key_val] = y_pred_fatal[j]
print(len(case_dict.keys()))
both_df["ConfirmedCases"] = both_df["state_date_key"].map(case_dict)
 both_df["Fatalities"] = both_df["state_date_key"].map(fatal_dict)
 null_count = both_df["Fatalities"].isnull().sum()
print("done")
 both_df.tail(10)
 test_df = pd.read_csv(PREFIX+"test.csv", parse_dates=["Date"])
 test_df = replace_state_nan(test_df)
 test_df.head(3)
 print(test_df.shape)
 merge_df = both_df[["Province_State","Date","ConfirmedCases","Fatalities"]].copy()
merge_df.sort_values(["Province_State","Date"], inplace=True)
 print(merge_df.head(5))
merge_df["ConfirmedCases"] = merge_df.groupby('Province_State')["ConfirmedCases"].cummax()
merge_df["Fatalities"] = merge_df.groupby('Province_State')["Fatalities"].cummax()
test_df = test_df.merge(merge_df, how="inner",
on=["Province_State","Date"])
 print(test_df.shape)
 test_df[["ForecastId","ConfirmedCases","Fatalities"]].to_csv("submission.csv", index=False)
 print("done")
<import_modules>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<categorify>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax')
<feature_engineering>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<save_to_csv>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
df_val = df_val_2
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission = submission.round({'ConfirmedCases': 0, 'Fatalities': 0})
 submission.to_csv('submission.csv', index=False)
 submission<load_from_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
train_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv" )<filter>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
india = train_df[train_df["Country_Region"]=="India"]<categorify>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
def province(state,country):
if state == "nan":
return country
return state<correct_missing_values>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train_df = train_df.fillna("nan" )<feature_engineering>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
train_df["Province_State"] = train_df.apply(lambda x: province(x["Province_State"],x["Country_Region"]),axis=1 )<prepare_x_and_y>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer="adam", metrics=[score])
 return model
italy = train_df[train_df["Country_Region"]=="Italy"]<import_modules>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
from datetime import datetime<data_type_conversions>
NFOLD = 7
kf = KFold(n_splits=NFOLD )
train_df["Date"] = pd.to_datetime(train_df["Date"] )<feature_engineering>
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=200, epochs=1000,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=500))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=500))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=500, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=500, verbose=0)/ NFOLD
train_df["month"] = train_df["Date"].dt.month<feature_engineering>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
train_df['day'] = train_df['Date'].dt.day<drop_column>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
train_df.drop('Date',axis=1,inplace=True )<import_modules>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
from sklearn import preprocessing<categorify>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
<categorify><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )