Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<save_to_csv>
import numpy as np
import pandas as pd
import os
import random
import matplotlib.pyplot as plt
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tqdm import tqdm
from PIL import Image
import pydicom
pred.to_csv('submission.csv',index=False )<import_modules>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.tsa.arima_model as arima
from sklearn.linear_model import LinearRegression
from sklearn import linear_model<set_options>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE = 128
warnings.filterwarnings('ignore')
 pd.set_option("display.max_rows", None, "display.max_columns", None )<load_from_csv>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
training_data_original = pd.read_csv('/kaggle/input/inputs3/train.csv')
 training_data_original['Date'] = pd.to_datetime(training_data_original['Date'])
 training_data = training_data_original[~training_data_original['Country_Region'].isin(['Diamond Princess', 'MS Zaandam'])]
countries = training_data['Country_Region'].unique()
countries_main = ['China', 'US', 'Australia', 'Canada']
states = training_data['Province_State'].unique()<filter>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
training_data[training_data['Country_Region'] == 'Belize']<load_from_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
state_metadata = pd.read_excel('/kaggle/input/externaldata3/states.xlsx')
 country_metadata = pd.read_excel('/kaggle/input/externaldata3/countries.xlsx' )<groupby>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
def data_preparation(training_data, countries, states, country_metadata, state_metadata, n_days_case,
n_days_fatal, min_num_cases = 2):
training_data_trun = training_data[training_data['ConfirmedCases'] >= min_num_cases]
conf_cases_dict = {}
fatal_dict = {}
for country in countries:
if country not in countries_main:
training_data_trun_loc = training_data_trun[(training_data_trun['Country_Region'] == country)&(pd.isnull(training_data_trun.Province_State)) ]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[country] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / country_metadata[country_metadata['Countries'] == country]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[country] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
for state in states:
training_data_trun_loc = training_data_trun[training_data_trun['Province_State'] == state]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[state] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / state_metadata[state_metadata['States'] == state]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[state] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
return pd.DataFrame(conf_cases_dict), pd.DataFrame(fatal_dict )<train_on_grid>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
def fts_training(input_df, rank = 3):
matrix = input_df.values
u, s, v = np.linalg.svd(matrix, full_matrices=False)
 scores = np.matmul(u[:, :rank], np.diag(s[:rank]))
pcs = v[:rank, :]
return scores, pcs<create_dataframe>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
def forecast_trajectories(training_data, countries, states, country_metadata,
state_metadata, loc = None, n_days_case = 30, n_days_fatal = 10,
forecast_days = 10, min_num_cases = 2, model_type = 'ARIMA',
components_modeling = True, rank = 3):
kf = KalmanFilter(initial_state_mean = np.zeros(rank ).tolist() , n_dim_obs = rank)
 conf_cases_df, fatal_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case,
n_days_fatal = n_days_case,
min_num_cases = min_num_cases)
 pred_countries = conf_cases_df.columns.tolist()
pred_countries_fatal = fatal_df.columns.tolist()
conf_cases_exog_df, fatal_exog_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case + forecast_days,
n_days_fatal = n_days_case + forecast_days,
min_num_cases = min_num_cases)
 scores_exog, pcs_exog = fts_training(conf_cases_exog_df, rank = rank)
 if len(scores_exog)> 0:
scores_exog = kf.em(scores_exog ).smooth(scores_exog)[0]
scores, pcs = fts_training(conf_cases_df, rank = rank)
 forecasted_scores = []
idx = 0
for score in scores.T:
if components_modeling:
exog = scores_exog[:n_days_case, idx] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, idx] if len(scores_exog)> 0 else None
y = score
else:
exog = scores_exog[:n_days_case, :] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, :] if len(scores_exog)> 0 else None
y = conf_cases_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
pred_traj = np.matmul(np.array(forecasted_scores ).T, pcs)
 pred_traj_df = pd.DataFrame(pred_traj, columns = pred_countries)
 for loc in pred_countries:
if loc in country_metadata['Countries'].values.tolist() :
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj_df[loc]
if loc in state_metadata['States'].values.tolist() :
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj_df[loc]
else:
pred_traj_df = pd.DataFrame()
if loc in countries:
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj
elif loc in states:
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj
fatal_scores_exog, fatal_pcs_exog = fts_training(fatal_exog_df, rank = rank)
 if len(fatal_pcs_exog)> 0:
fatal_scores_exog = kf.em(fatal_scores_exog ).smooth(fatal_scores_exog)[0]
fatal_scores, fatal_pcs = fts_training(fatal_df, rank = rank)
 forecasted_fatal_scores = []
idx = 0
for fatal_score in fatal_scores.T:
if components_modeling:
exog = fatal_scores_exog[:n_days_fatal, idx] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, idx] if len(fatal_scores_exog)> 0 else None
y = fatal_score
else:
exog = fatal_scores_exog[:n_days_fatal, :] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, :] if len(fatal_scores_exog)> 0 else None
y = fatal_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
fatal_pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_fatal_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
fatal_pred_traj = np.matmul(np.array(forecasted_fatal_scores ).T, fatal_pcs)
 fatal_pred_traj_df = pd.DataFrame(fatal_pred_traj, columns = pred_countries_fatal)
 else:
fatal_pred_traj_df = pd.DataFrame()
if loc in countries:
fatal_pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * fatal_pred_traj
elif loc in states:
fatal_pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * fatal_pred_traj
return pred_traj_df, fatal_pred_traj_df 
<filter>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
def generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases):
for country in countries:
print(country)
 if country not in countries_main and country not in excl_list:
test_loc_df = test_data[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) ].reset_index()
train_loc_df = training_data[(training_data['Country_Region'] == country)&(pd.isnull(training_data.Province_State)) ].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 min_num_cases_temp = min_num_cases
if len(effective_df)> 0:
while min_num_cases_temp > 1:
try:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases_temp].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = country,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases_temp,
rank = 4)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'Fatalities'] = test_loc_df['Fatalities'].values
break
except:
min_num_cases_temp = math.floor(min_num_cases_temp / 2)
 continue
for state in states:
print(state)
 if str(state)not in excl_list:
test_loc_df = test_data[(test_data['Province_State'] == state)].reset_index()
train_loc_df = training_data[(training_data['Province_State'] == state)].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 min_num_cases_temp = min_num_cases
if len(effective_df)> 0:
while min_num_cases_temp > 1:
try:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases_temp].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = state,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases_temp,
rank = 4)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'Fatalities'] = test_loc_df['Fatalities'].values
break
except:
min_num_cases_temp = math.floor(min_num_cases_temp / 2)
 continue
return test_data 
<load_from_csv>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
test_data = pd.read_csv('/kaggle/input/inputs3/test.csv')
 test_data['Date'] = pd.to_datetime(test_data['Date'] )<feature_engineering>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
test_data['ConfirmedCases'] = None
test_data['Fatalities'] = None<define_variables>
LR = 0.1
BETA_1 = 0.9
BETA_2 = 0.999
DECAY = 0.01
AMSGRAD = False
excl_list = ['Belize', 'Botswana', 'Diamond Princess', 'MS Zaandam', 'Angola',
'Turks and Caicos Islands', 'Burma', 'Burundi', 'Chad', 'Eritrea',
'Grenada', 'Guinea-Bissau', 'Holy See', 'Kosovo',
'Laos', 'Libya', 'Mali', 'Mozambique', 'Saint Kitts and Nevis', 'Somalia',
'Syria', 'nan', 'Saint Barthelemy', 'Virgin Islands', 'Montserrat',
'Saint Vincent and the Grenadines', 'Sierra Leone', 'Northwest Territories',
'Yukon', 'Anguilla', 'British Virgin Islands', 'Papua New Guinea', 'Bhutan',
'Congo(Brazzaville)', 'Gabon', 'Guinea', 'Guyana', 'Haiti', 'Namibia',
'Saint Lucia', 'Seychelles', 'Curacao', 'Cayman Islands',
'Central African Republic', 'Liberia', 'Mauritania', 'Nepal',
'Nicaragua', 'Sudan']<predict_on_test>
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="NN-Optimized")
 model.compile(loss=mloss(0.8), optimizer = tf.keras.optimizers.Adam(lr = LR, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = None, decay = DECAY, amsgrad = AMSGRAD), metrics=[score])
 return model
predictions = generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases = 100)
<split>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
def fill_excl_pred(predictions, training_data_original, test_data):
for country in missing_countries:
print(country)
 pred_loc_df = predictions[predictions['Country_Region'] == country]
train_loc_df = training_data_original[training_data_original['Country_Region'] == country]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Country_Region'] == country]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['Fatalities']
if len(series_comf_cases)> 0:
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
else:
comf_cases_pred = []
fatal_pred = []
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Country_Region'] == country, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Country_Region'] == country, 'Fatalities'] = fatal_loc.astype(int)
 for state in missing_states:
print(state)
 pred_loc_df = predictions[predictions['Province_State'] == state]
train_loc_df = training_data_original[training_data_original['Province_State'] == state]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Province_State'] == state]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['Fatalities']
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Province_State'] == state, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Province_State'] == state, 'Fatalities'] = fatal_loc.astype(int)
 return predictions 
<predict_on_test>
NFOLD = 10
kf = KFold(n_splits=NFOLD )
prediction_final = fill_excl_pred(predictions, training_data_original, test_data )<filter>
%%time
count = 0
for train_idx, val_idx in kf.split(z):
count += 1
print(f"FOLD {count}:")
 net = make_model()
net.fit(z[train_idx], y[train_idx], batch_size=BATCH_SIZE, epochs=850,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("Train:", net.evaluate(z[train_idx], y[train_idx], verbose=0, batch_size=BATCH_SIZE))
print("Val:", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("Predicting Test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
prediction_final[prediction_final['Country_Region'] == 'Germany']<data_type_conversions>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
predictions_csv = pd.DataFrame()
predictions_csv['ForecastId'] = prediction_final['ForecastId'].astype(int)
 predictions_csv['ConfirmedCases'] = prediction_final['ConfirmedCases'].astype(float)
 predictions_csv['Fatalities'] = prediction_final['Fatalities'].astype(float )<save_to_csv>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
predictions_csv.to_csv('submission.csv', index = False )<import_modules>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from itertools import cycle, islice
import seaborn as sb
import matplotlib.dates as dates
from datetime import datetime
from sklearn import preprocessing
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split<load_from_csv>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv' )<count_values>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<sort_values><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )