Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<set_options>
%matplotlib inline
del dataset
gc.collect()<split>
def dS_dt(S, I, R_t, T_inf):
return -(R_t / T_inf)* I * S
def dE_dt(S, E, I, R_t, T_inf, T_inc):
return(R_t / T_inf)* I * S -(T_inc**-1)* E
def dI_dt(I, E, T_inc, T_inf):
return(T_inc**-1)* E -(T_inf**-1)* I
def dR_dt(I, T_inf):
return(T_inf**-1)* I
def SEIR_model(t, y, R_t, T_inf, T_inc):
if callable(R_t):
reproduction = R_t(t)
 else:
reproduction = R_t
S, E, I, R = y
S_out = dS_dt(S, I, reproduction, T_inf)
 E_out = dE_dt(S, E, I, reproduction, T_inf, T_inc)
 I_out = dI_dt(I, E, T_inc, T_inf)
 R_out = dR_dt(I, T_inf)
 return [S_out, E_out, I_out, R_out]
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 train['Date_datetime'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')) )
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
pop_info = pd.read_csv('/kaggle/input/covid19-population-data/population_data.csv')
 country_pop = pop_info.query('Type == "Country/Region"')
 province_pop = pop_info.query('Type == "Province/State"')
 country_lookup = dict(zip(country_pop['Name'], country_pop['Population']))
province_lookup = dict(zip(province_pop['Name'], province_pop['Population']))
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
import riiideducation
import matplotlib.pyplot as plt
import seaborn as sns
import random
import os
import pickle<load_pretrained>
Country = 'Hubei'
N = pop_info[pop_info['Name']==Country]['Population'].tolist() [0]
train_loc = train[train['Country_Region']==Country].query('ConfirmedCases > 0')
 if len(train_loc)==0:
train_loc = train[train['Province_State']==Country].query('ConfirmedCases > 0')
 n_infected = train_loc['ConfirmedCases'].iloc[0]
max_days = len(train_loc)
 s =(N - n_infected)/ N
e = 0.
 i = n_infected / N
r = 0.
 T_inc = 5.2
T_inf = 2.9
R_0 = 3.954
sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(R_0, T_inf, T_inc),
t_eval=np.arange(max_days))
plot_model_and_predict(train_loc, N, sol, title = 'SEIR Model(without intervention)' )
Parameters = pd.read_pickle('.. /input/the-best-of-all-parameters-on-lightgbm/The_best_of_all_parameters_on_LightGBM.pickle')
 print('The result of Optuna:')
 print(Parameters)
 del Parameters<feature_engineering>
from scipy.optimize import minimize
from sklearn.metrics import mean_squared_log_error, mean_squared_error
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED)
 def add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = True):
answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)
 explanation_u_avg = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)
 answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)
 explanation_q_avg = np.zeros(len(df), dtype = np.float32)
 answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)
 for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):
if answered_correctly_u_count[row[0]] != 0:
answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]
elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]
explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]
else:
answered_correctly_u_avg[num] = np.nan
elapsed_time_u_avg[num] = np.nan
explanation_u_avg[num] = np.nan
if len(timestamp_u[row[0]])== 0:
timestamp_u_recency_1[num] = np.nan
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 1:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 2:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 3:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]
if len(timestamp_u_incorrect[row[0]])== 0:
timestamp_u_incorrect_recency[num] = np.nan
else:
timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]
if answered_correctly_q_count[row[2]] != 0:
answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]
elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]
explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]
else:
answered_correctly_q_avg[num] = np.nan
elapsed_time_q_avg[num] = np.nan
explanation_q_avg[num] = np.nan
answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]
answered_correctly_u_count[row[0]] += 1
elapsed_time_u_sum[row[0]] += row[3]
explanation_u_sum[row[0]] += int(row[4])
 if len(timestamp_u[row[0]])== 3:
timestamp_u[row[0]].pop(0)
 timestamp_u[row[0]].append(row[5])
 else:
timestamp_u[row[0]].append(row[5])
 answered_correctly_q_count[row[2]] += 1
elapsed_time_q_sum[row[2]] += row[3]
explanation_q_sum[row[2]] += int(row[4])
 answered_correctly_uq[row[0]][row[2]] += 1
if update:
answered_correctly_u_sum[row[0]] += row[1]
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[5])
 else:
timestamp_u_incorrect[row[0]].append(row[5])
 answered_correctly_q_sum[row[2]] += row[1]
user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg,
'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg,
'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,
'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})
 df = pd.concat([df, user_df], axis = 1)
 return df
def update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):
for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:
if row[3] == 0:
answered_correctly_u_sum[row[0]] += row[1]
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[4])
 else:
timestamp_u_incorrect[row[0]].append(row[4])
 answered_correctly_q_sum[row[2]] += row[1]
return
def read_and_preprocess(feature_engineering = False):
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
if feature_engineering:
train = train.iloc[-40000000:]
train = train.loc[train.content_type_id == False].reset_index(drop = True)
 valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8')
 prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()
train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 questions_df = pd.read_csv(question_file)
 questions_df['part'] = questions_df['part'].astype(np.int32)
 questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)
 train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 answered_correctly_u_count = defaultdict(int)
 answered_correctly_u_sum = defaultdict(int)
 elapsed_time_u_sum = defaultdict(int)
 explanation_u_sum = defaultdict(int)
 timestamp_u = defaultdict(list)
 timestamp_u_incorrect = defaultdict(list)
 answered_correctly_q_count = defaultdict(int)
 answered_correctly_q_sum = defaultdict(int)
 elapsed_time_q_sum = defaultdict(int)
 explanation_q_sum = defaultdict(int)
 answered_correctly_uq = defaultdict(lambda: defaultdict(int))
print('User feature calculation started...')
 print('
')
 train = add_features(train, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)
 valid = add_features(valid, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)
 gc.collect()
print('User feature calculation completed...')
 print('
')
 features_dicts = {
'answered_correctly_u_count': answered_correctly_u_count,
'answered_correctly_u_sum': answered_correctly_u_sum,
'elapsed_time_u_sum': elapsed_time_u_sum,
'explanation_u_sum': explanation_u_sum,
'answered_correctly_q_count': answered_correctly_q_count,
'answered_correctly_q_sum': answered_correctly_q_sum,
'elapsed_time_q_sum': elapsed_time_q_sum,
'explanation_q_sum': explanation_q_sum,
'answered_correctly_uq': answered_correctly_uq,
'timestamp_u': timestamp_u,
'timestamp_u_incorrect': timestamp_u_incorrect
}
return train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts
def train_and_evaluate(train, valid, feature_engineering = False):
TARGET = 'answered_correctly'
FEATURES = ['prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',
'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3',
'timestamp_u_incorrect_recency']
if feature_engineering:
train = train.sample(15000000, random_state = SEED)
 gc.collect()
print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')
 drop_cols = list(set(train.columns)- set(FEATURES))
y_train = train[TARGET]
y_val = valid[TARGET]
train.drop(drop_cols, axis = 1, inplace = True)
 valid.drop(drop_cols, axis = 1, inplace = True)
 gc.collect()
lgb_train = lgb.Dataset(train[FEATURES], y_train)
 lgb_valid = lgb.Dataset(valid[FEATURES], y_val)
 del train, y_train
gc.collect()
params = pd.read_pickle('.. /input/the-best-of-all-parameters-on-lightgbm/The_best_of_all_parameters_on_LightGBM.pickle')
 model = lgb.train(
 params = params,
train_set = lgb_train,
num_boost_round = 10000,
valid_sets = [lgb_train, lgb_valid],
early_stopping_rounds = 10,
verbose_eval = 50 
)
 print('Our ROC AUC score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES], num_iteration=model.best_iteration)))
 feature_importance = model.feature_importance()
feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance} ).sort_values('Importance', ascending = False)
 fig = plt.figure(figsize =(10, 10))
fig.suptitle('Feature Importance', fontsize = 20)
 plt.tick_params(axis = 'x', labelsize = 12)
 plt.tick_params(axis = 'y', labelsize = 12)
 plt.xlabel('Importance', fontsize = 15)
 plt.ylabel('Features', fontsize = 15)
 sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')
 plt.show()
return TARGET, FEATURES, model
def inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):
answered_correctly_u_count = features_dicts['answered_correctly_u_count']
answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']
elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']
explanation_u_sum = features_dicts['explanation_u_sum']
answered_correctly_q_count = features_dicts['answered_correctly_q_count']
answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']
elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']
explanation_q_sum = features_dicts['explanation_q_sum']
answered_correctly_uq = features_dicts['answered_correctly_uq']
timestamp_u = features_dicts['timestamp_u']
timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df[TARGET] = 0
test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)
 test_df[TARGET] = model.predict(test_df[FEATURES], num_iteration=model.best_iteration)
 set_predict(test_df[['row_id', TARGET]])
 pickle.dump(model, open('The_best_of_model_on_LightGBM.pickle', 'wb'))
print('Save the best model of LightGBM.')
 print('Job Done')
 train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = True)
 TARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = True)
 inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts )<import_modules>
def eval_model_const(params, data, population, return_solution=False, forecast_days=0):
R_0, cfr = params
N = population
n_infected = data['ConfirmedCases'].iloc[0]
max_days = len(data)+ forecast_days
s, e, i, r =(N - n_infected)/ N, 0, n_infected / N, 0
def time_varying_reproduction(t):
if t > 80:
return R_0 * 0.5
else:
return R_0
sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(time_varying_reproduction, T_inf, T_inc),
t_eval=np.arange(0, max_days))
sus, exp, inf, rec = sol.y
y_pred_cases = np.clip(( inf + rec)* N ,0,np.inf)
 y_true_cases = data['ConfirmedCases'].values
y_pred_fat = np.clip(rec*N* cfr, 0, np.inf)
 y_true_fat = data['Fatalities'].values
optim_days = min(20, len(data))
weights = 1 / np.arange(1, optim_days+1)[::-1]
msle_cases = mean_squared_log_error(y_true_cases[-optim_days:], y_pred_cases[-optim_days:], weights)
 msle_fat = mean_squared_log_error(y_true_fat[-optim_days:], y_pred_fat[-optim_days:], weights)
 msle_final = np.mean([msle_cases, msle_fat])
 if return_solution:
return msle_final, sol
else:
return msle_final
import gc
import random
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader<define_variables>
def eval_model_decay(params, data, population, return_solution=False, forecast_days=0):
R_0, cfr, k, L = params
N = population
n_infected = data['ConfirmedCases'].iloc[0]
max_days = len(data)+ forecast_days
s, e, i, r =(N - n_infected)/ N, 0, n_infected / N, 0
def time_varying_reproduction(t):
return R_0 /(1 +(t/L)**k)
 sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(time_varying_reproduction, T_inf, T_inc),
t_eval=np.arange(0, max_days))
sus, exp, inf, rec = sol.y
y_pred_cases = np.clip(( inf + rec)* N ,0,np.inf)
 y_true_cases = data['ConfirmedCases'].values
y_pred_fat = np.clip(rec*N* cfr, 0, np.inf)
 y_true_fat = data['Fatalities'].values
optim_days = min(20, len(data))
weights = 1 / np.arange(1, optim_days+1)[::-1]
msle_cases = mean_squared_log_error(y_true_cases[-optim_days:], y_pred_cases[-optim_days:], weights)
 msle_fat = mean_squared_log_error(y_true_fat[-optim_days:], y_pred_fat[-optim_days:], weights)
 msle_final = np.mean([msle_cases, msle_fat])
 if return_solution:
return msle_final, sol
else:
return msle_final
MAX_SEQ = 160 
<load_from_csv>
def fit_model_new(data, area_name, initial_guess=[2.2, 0.02, 2, 50],
bounds=(( 1, 20),(0, 0.15),(1, 3),(1, 100)) , make_plot=True, decay_mode = None):
if area_name in ['France']:
train = data.query('ConfirmedCases > 0' ).copy() [:-1]
else:
train = data.query('ConfirmedCases > 0' ).copy()
train_data = train
if len(train_data)== 0:
result_zero = np.zeros(( 43))
return pd.DataFrame({'ConfirmedCases':result_zero,'Fatalities':result_zero}), 0
try:
population = pop_info[pop_info['Name']==area_name]['Population'].tolist() [0]
except IndexError:
print('country not in population set, '+str(area_name))
population = 1000000
if area_name == 'US':
population = 327200000
if area_name == 'Global':
population = 7744240900
cases_per_million = train_data['ConfirmedCases'].max() * 10**6 / population
n_infected = train_data['ConfirmedCases'].iloc[0]
if cases_per_million < 1:
population = population/100
res_const = minimize(eval_model_const, [2.2, 0.02], bounds=(( 1, 20),(0, 0.15)) ,
args=(train_data, population, False),
method='L-BFGS-B')
 res_decay = minimize(eval_model_decay, initial_guess, bounds=bounds,
args=(train_data, population, False),
method='L-BFGS-B')
 test_end = datetime.datetime.strptime('2020-05-14','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-04-02','%Y-%m-%d')
 train_test = data[data.Date_datetime>=test_start]
test_period =(test_end - test_start ).days
train_max = train_data.Date_datetime.max()
train_min = train_data.Date_datetime.min()
add_date = 0
delta_days =(test_end - train_max ).days
train_add_time=[]
if train_min > test_start:
add_date =(train_min-test_start ).days
last = train_min-timedelta(add_date)
 train_add_time = np.arange(last, train_min, dtype='datetime64[D]' ).tolist()
train_add_time = pd.to_datetime(train_add_time)
 dates_all = train_add_time.append(pd.to_datetime(np.arange(train_min, test_end+timedelta(1), dtype='datetime64[D]')))
 else:
dates_all = pd.to_datetime(np.arange(train_min, test_end+timedelta(1), dtype='datetime64[D]'))
if decay_mode is None:
if res_const.fun < res_decay.fun :
msle, sol = eval_model_const(res_const.x, train_data, population, True, delta_days+add_date)
 res = res_const
else:
msle, sol = eval_model_decay(res_decay.x, train_data, population, True, delta_days+add_date)
 res = res_decay
R_0, cfr, k, L = res.x
else:
if decay_mode =='day_decay':
msle, sol = eval_model_const(res_const.x, train_data, population, True, delta_days+add_date)
 res = res_const
else:
msle, sol = eval_model_decay(res_decay.x, train_data, population, True, delta_days+add_date)
 res = res_decay
R_0, cfr, k, L = res.x
sus, exp, inf, rec = sol.y
y_pred = pd.DataFrame({
'ConfirmedCases': cumsum_signal(np.diff(( inf + rec)* population, prepend=n_infected ).cumsum()),
'Fatalities': cumsum_signal(( np.clip(rec * population * res.x[1], 0, np.inf)).tolist())
 })
 y_pred_valid = y_pred.iloc[:len(train_data)]
y_pred_test = pd.concat([train_test[['ConfirmedCases', 'Fatalities']],y_pred.iloc[-(delta_days):]], ignore_index=True)
 y_true_valid = train_data[['ConfirmedCases', 'Fatalities']]
valid_msle_cases = mean_squared_log_error(y_true_valid['ConfirmedCases'], y_pred_valid['ConfirmedCases'])
 valid_msle_fat = mean_squared_log_error(y_true_valid['Fatalities'], y_pred_valid['Fatalities'])
 valid_msle = np.mean([valid_msle_cases, valid_msle_fat])
 if make_plot:
if len(res.x)<=2:
print(f'Validation MSLE: {valid_msle:0.5f}, using intervention days decay, Reproduction number(R0): {res.x[0]:0.5f}, Fatal rate : {res.x[1]:0.5f}')
 else:
print(f'Validation MSLE: {valid_msle:0.5f}, using Hill decay, Reproduction number(R0): {res.x[0]:0.5f}, Fatal rate : {res.x[1]:0.5f}, K : {res.x[2]:0.5f}, L: {res.x[3]:0.5f}')
 f = plt.figure(figsize=(16,5))
ax = f.add_subplot(1,2,1)
 ax.plot(exp, 'y', label='Exposed');
ax.plot(inf, 'r', label='Infected');
ax.plot(rec, 'c', label='Recovered/deceased');
plt.title('SEIR Model Trends')
 plt.xlabel("Days", fontsize=10);
plt.ylabel("Fraction of population", fontsize=10);
plt.legend(loc='best');
ax2 = f.add_subplot(1,2,2)
 xaxis = train_data['Date_datetime'].tolist()
xaxis = dates.date2num(xaxis)
 hfmt = dates.DateFormatter('%m
%d')
 ax2.xaxis.set_major_formatter(hfmt)
 ax2.plot(np.array(train_data['Date_datetime'], dtype='datetime64[D]'),train_data['ConfirmedCases'],label='Confirmed Cases(train)', c='g')
 ax2.plot(np.array(train_data['Date_datetime'], dtype='datetime64[D]'), y_pred['ConfirmedCases'][:len(train_data)],label='Cumulative modeled infections', c='r')
 plt.title('Real ConfirmedCase and Predict ConfirmedCase')
 plt.legend(loc='best');
plt.show()
if len(res.x)>2:
msle, sol = eval_model_decay(res.x, train_data, population, True, 300)
 else:
msle, sol = eval_model_const(res.x, train_data, population, True, 300)
 sus, exp, inf, rec = sol.y
y_pred = pd.DataFrame({
'ConfirmedCases': cumsum_signal(np.diff(( inf + rec)* population, prepend=n_infected ).cumsum()),
'Fatalities': cumsum_signal(np.clip(rec, 0, np.inf)* population * res.x[1])
 })
 start = train_min
end = start + timedelta(len(y_pred))
time_array = np.arange(start, end, dtype='datetime64[D]')
 max_day = numpy.where(inf == numpy.amax(inf)) [0][0]
where_time = time_array[max_day]
pred_max_day = y_pred['ConfirmedCases'][max_day]
xy_show_max_estimation =(where_time, max_day)
 con = y_pred['ConfirmedCases']
fat = y_pred['Fatalities']
max_day_con = numpy.where(con == numpy.amax(con)) [0][0]
max_day_fat = numpy.where(fat == numpy.amax(fat)) [0][0]
max_con = numpy.amax(con)
 max_fat = numpy.amax(fat)
 where_time_con = time_array[len(time_array)-50]
xy_show_max_estimation_confirmed =(where_time_con, max_con)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=time_array, y=y_pred['ConfirmedCases'].astype(int),
mode='lines',
line = dict(color='red'),
name='Estimation Confirmed Case Start from '+ str(start.date())+ ' to ' +str(end.date())))
 fig.add_trace(go.Scatter(x=time_array, y=y_pred['Fatalities'].astype(int),
mode='lines',
line = dict(color='yellow'),
name='Estimation Fatalities Start from '+ str(start.date())+ ' to ' +str(end.date())))
 fig.add_trace(go.Scatter(x=time_array[:len(train)], y=train['ConfirmedCases'],
mode='lines',
name='Confirmed case until '+ str(train_max.date()),line = dict(color='green', width=4)))
 fig.add_trace(go.Scatter(x=time_array[:len(train)], y=train['Fatalities'],
mode='lines',
name='Fatalities case until '+ str(train_max.date()),line = dict(color='blue', width=4)))
 fig.add_annotation(
 x=where_time_con,
y=max_con-(max_con/30),
showarrow=False,
text="Estimate Max Case around:" +str(int(max_con)) ,
font=dict(
 color="Blue",
size=15 
))
fig.add_annotation(
 x=where_time_con,
y=max_fat-(max_fat/30),
showarrow=False,
text="Estimate Max death around:" +str(int(max_fat)) ,
font=dict(
 color="Blue",
size=15 
))
fig.add_annotation(
 x=time_array[len(train)-1],
y=train['ConfirmedCases'].tolist() [-1],
showarrow=True,
text=f"Real Max ConfirmedCase: " +str(int(train['ConfirmedCases'].tolist() [-1])))
 fig.add_annotation(
 x=time_array[len(train)-1],
y=train['Fatalities'].tolist() [-1],
showarrow=True,
text=f"Real Max Fatalities: " +str(int(train['Fatalities'].tolist() [-1])))
 fig.add_annotation(
 x=where_time,
y=pred_max_day,
text='Infect start decrease from: ' + str(where_time))
fig.update_layout(title='Estimate Confirmed Case ,'+area_name+' Total population ='+ str(int(population)) , legend_orientation="h")
 fig.show()
return y_pred_test, valid_msle
%%time
dtype = {'timestamp':'int64',
'user_id':'int32' ,
'content_id':'int16',
'content_type_id':'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)
 train_df.head()<sort_values>
country = 'Taiwan*'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )<count_unique_values>
country = 'Japan'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))<groupby>
country = 'Italy'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
del train_df
gc.collect()<define_variables>
country = 'New York'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )
random.seed(1)
<define_variables>
country = 'Afghanistan'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label<create_dataframe>
country = 'US'
country_pd_train = train[train['Country_Region']==country]
country_pd_train2 = country_pd_train.groupby(['Date'] ).sum().reset_index()
country_pd_train2['Date_datetime'] = country_pd_train2['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 a,b = fit_model_new(country_pd_train2,country,make_plot=True )
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
<define_search_model>
country = 'Global'
country_pd_train2 = train.groupby(['Date'] ).sum().reset_index()
country_pd_train2['Date_datetime'] = country_pd_train2['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 a,b = fit_model_new(country_pd_train2,country,make_plot=True )
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<choose_model_class>
validation_scores = []
validation_county = []
validation_country = []
for country in tqdm(train['Country_Region'].unique()):
country_pd_train = train[train['Country_Region']==country]
if len(country_pd_train['Province_State'].unique())<2:
predict_test, score = fit_model_new(country_pd_train,country,make_plot=False)
 if score ==0:
print(f'{country} no case')
 validation_scores.append(score)
 validation_county.append(country)
 validation_country.append(country)
 test.loc[test['Country_Region']==country,'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[test['Country_Region']==country,'Fatalities'] = predict_test['Fatalities'].tolist()
else:
for state in country_pd_train['Province_State'].unique() :
if state != state:
state_pd = country_pd_train[country_pd_train['Province_State'].isna() ]
predict_test, score = fit_model_new(state_pd,state,make_plot=False)
 if score ==0:
print(f'{country} / {state} no case')
 validation_scores.append(score)
 validation_county.append(state)
 validation_country.append(country)
 test.loc[(test['Country_Region']==country)&(test['Province_State'].isna()),'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[(test['Country_Region']==country)&(test['Province_State'].isna()),'Fatalities'] = predict_test['Fatalities'].tolist()
else:
state_pd = country_pd_train[country_pd_train['Province_State']==state]
predict_test, score = fit_model_new(state_pd,state,make_plot=False)
 if score ==0:
print(f'{country} / {state} no case')
 validation_scores.append(score)
 validation_county.append(state)
 validation_country.append(country)
 test.loc[(test['Country_Region']==country)&(test['Province_State']==state),'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[(test['Country_Region']==country)&(test['Province_State']==state),'Fatalities'] = predict_test['Fatalities'].tolist()
print(f'Mean validation score: {np.average(validation_scores):0.5f}' )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=256)
 optimizer = torch.optim.Adam(model.parameters() , lr=1.5e-3)
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 8e-3, eta_min=0, last_epoch=-1)
 model.to(device)
 criterion.to(device )<train_model>
validation_scores = pd.DataFrame({'country/state':validation_country,'country':validation_county,'MSLE':validation_scores})
 validation_scores.sort_values(by=['MSLE'], ascending=False ).head(20 )
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
scheduler.step()
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc<train_model>
large_msle = validation_scores[validation_scores['MSLE']>1]
epochs = 35
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))<save_model>
test_end = datetime.datetime.strptime('2020-05-14','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-04-02','%Y-%m-%d')
 train_max = train.Date_datetime.max()
train_min = train.Date_datetime.min()
delta_days =(test_end - train_max ).days
all_days =(test_end - train_min ).days
delta_days,all_days
torch.save(model.state_dict() , "SAKT-HDKIM.pt" )<set_options>
for country in large_msle['country'].unique() :
if(country!= country)==False:
country_pd_train = train[train['Country_Region']==country]
country_pd_test = test[test['Country_Region']==country]
if len(country_pd_train)==0:
country_pd_train = train[train['Province_State']==country]
country_pd_test = test[test['Province_State']==country]
test_end = datetime.datetime.strptime('2020-05-14','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-04-02','%Y-%m-%d')
 train_max = train.Date_datetime.max()
train_min = train.Date_datetime.min()
delta_days =(test_end - train_max ).days
x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [:-7]
valid_x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [-7:]
y = country_pd_train['ConfirmedCases'][:-7]
valid_y = country_pd_train['ConfirmedCases'][-7:]
y_fat = country_pd_train['Fatalities'][:-7]
valid_y_fat = country_pd_train['Fatalities'][-7:]
model = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model = model.fit(x, y)
 model_fat = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model_fat = model_fat.fit(x, y_fat)
 predict_y = model.predict(valid_x)
 predict_yfat = model_fat.predict(valid_x)
 score = mean_squared_log_error(np.clip(valid_y,0,np.inf), np.clip(predict_y,0,np.inf))
score_fat = mean_squared_log_error(np.clip(valid_y_fat,0,np.inf), np.clip(predict_yfat,0,np.inf))
score =(score+score_fat)/2
print(f'{country} {score:0.5f}')
 if score < large_msle[large_msle['country']==country]['MSLE'].tolist() [0]:
validation_scores.loc[validation_scores['country']==country,'MSLE'] = score
predict_x =(np.array(range(all_days)) ).reshape(( -1,1))
predict_result = model.predict(predict_x)
 predict_fat_result = model_fat.predict(predict_x)
 test.loc[test['Province_State']==country,'ConfirmedCases'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['ConfirmedCases'].tolist() +predict_result[-(delta_days):].tolist()
test.loc[test['Province_State']==country,'Fatalities'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['Fatalities'].tolist() +predict_fat_result[-(delta_days):].tolist()
else:
x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [:-7]
valid_x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [-7:]
y = country_pd_train['ConfirmedCases'][:-7]
valid_y = country_pd_train['ConfirmedCases'][-7:]
y_fat = country_pd_train['Fatalities'][:-7]
valid_y_fat = country_pd_train['Fatalities'][-7:]
model = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model = model.fit(x, y)
 model_fat = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model_fat = model_fat.fit(x, y_fat)
 predict_y = model.predict(valid_x)
 predict_yfat = model_fat.predict(valid_x)
 score = mean_squared_log_error(np.clip(valid_y,0,np.inf), np.clip(predict_y,0,np.inf))
score_fat = mean_squared_log_error(np.clip(valid_y_fat,0,np.inf), np.clip(predict_yfat,0,np.inf))
score =(score+score_fat)/2
print(f'{country} {score:0.5f}')
 if score < large_msle[large_msle['country']==country]['MSLE'].tolist() [0]:
validation_scores.loc[validation_scores['country']==country,'MSLE'] = score
predict_x =(np.array(range(all_days)) ).reshape(( -1,1))
predict_result = model.predict(predict_x)
 predict_fat_result = model_fat.predict(predict_x)
 test.loc[test['Country_Region']==country,'ConfirmedCases'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['ConfirmedCases'].tolist() +predict_result[-(delta_days):].tolist()
test.loc[test['Country_Region']==country,'Fatalities'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['Fatalities'].tolist() +predict_fat_result[-(delta_days):].tolist()
MSLE_score = validation_scores['MSLE'].tolist()
print(f'Mean validation score: {np.average(MSLE_score):0.5f}')
del dataset
gc.collect()<split>
submit = pd.read_csv('.. /input/covid19-global-forecasting-week-4/submission.csv')
 submit['Fatalities'] = test['Fatalities'].astype('float')
 submit['ConfirmedCases'] = test['ConfirmedCases'].astype('float')
 submit.to_csv('submission.csv',index=False )