Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<load_from_csv>
np.random.seed(1337)
 base_folder = '/kaggle/input/'
plt.rcParams['figure.figsize'] = [15, 7]
ext_cols = ['LaborForceTotal', 'LaborForcePerCapita', 'DeathRate', 'AirTrafficPassengersTotal',
'AirTrafficPassengersPerCapita', 'HospitalBedDensity', 'Obesity', 'OldPeople',
'PhysiciansDensity', 'AlcoholConsumptionPerCapita', 'CigaretteConsumptionPerCapita']
pop_cols = ['Yearly change', 'Density', 'Land Area', 'Med.Age', 'Urban Pop', 'Population']
add_cols = ['DayNum', 'PreviousDay-0ExposedDensity']
data_base = base_folder + 'covid19-global-forecasting-week-4/'
df = pd.read_csv(data_base + 'train.csv' ).drop(columns=['Id'])
 df.rename(columns={'Province_State': 'Province/State', 'Country_Region': 'Country/Region'}, inplace=True)
 df['Province/State'].fillna('entire country', inplace=True)
 df['ConfirmedCasesDelta'] = df.groupby(['Country/Region', 'Province/State'])['ConfirmedCases'].diff().fillna(0)
 df['FatalitiesDelta'] = df.groupby(['Country/Region', 'Province/State'])['Fatalities'].diff().fillna(0)
 df['Date'] = pd.to_datetime(df['Date'] ).dt.date
day_zero = min(df['Date'])
 df['DayNum'] =(df['Date'] - day_zero ).apply(lambda x: int(x.days))
df
%%time
pd.read_sql('SELECT * from users LIMIT 5', conn )<create_dataframe>
def add_extra_features_from_previous_days(data_fr, tail_size=5):
cols_tmp = []
col_prefix = 'PreviousDay'
for i in range(0, tail_size):
col_cc = '{}-{}ConfirmedCases'.format(col_prefix, i)
 col_f = '{}-{}Fatalities'.format(col_prefix, i)
 data_fr[col_cc] = data_fr.groupby(['Country/Region', 'Province/State'])['ConfirmedCases'].shift(periods=i+1, fill_value=0)
 data_fr[col_f] = data_fr.groupby(['Country/Region', 'Province/State'])['Fatalities'].shift(periods=i+1, fill_value=0)
 data_fr[col_cc + 'Delta'] = data_fr.groupby(['Country/Region', 'Province/State'])[col_cc].diff().fillna(0)
 data_fr[col_f + 'Delta'] = data_fr.groupby(['Country/Region', 'Province/State'])[col_f].diff().fillna(0)
 cols_tmp += [col_cc, col_f, col_cc + 'Delta', col_f + 'Delta']
return cols_tmp
%%time
q_cols = [
'question_id',
'part',
'tag__0',
'part_correct_pct',
'tag__0_correct_pct',
'question_id_correct_pct'
]
df_questions[q_cols].to_sql('questions', conn, method='multi', index=False)
 _ = conn.execute('CREATE UNIQUE INDEX question_id_index ON questions(question_id)')
 del df_questions
gc.collect()<load_from_csv>
TAIL = 50
previous_days_cols = add_extra_features_from_previous_days(df, TAIL)
 df
%%time
pd.read_sql('SELECT * from questions LIMIT 5', conn )<load_from_csv>
def special_day_CC(org_df, number_of_cc):
print('calculating for every Country & Province days passed from the first day when ConfirmedCases >= {}'.format(number_of_cc))
col_final = 'Day_CC{}'.format(number_of_cc)
 col = 'Day_CC{}_zero'.format(number_of_cc)
 org_df[col] = org_df.where(
(org_df['ConfirmedCases'] >= number_of_cc)& 
(( org_df['PreviousDay-0ConfirmedCases'] < number_of_cc)|(org_df['Date'] == day_zero)) 
)['DayNum']
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].ffill()
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].bfill()
day_num = org_df['DayNum'] - org_df[col] + 1
org_df[col_final] =(day_num - day_num.where(day_num<0 ).fillna(0)).fillna(0)
 return col, col_final
def special_day_F(org_df, number_of_f):
print('calculating for every Country & Province days passed from the first day when Fatalities >= {}'.format(number_of_f))
col_final = 'Day_F{}'.format(number_of_f)
 col = 'Day_F{}_zero'.format(number_of_f)
 org_df[col] = org_df.where(
(org_df['Fatalities'] >= number_of_f)& 
(( org_df['PreviousDay-0Fatalities'] < number_of_f)|(org_df['Date'] == day_zero)) 
)['DayNum']
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].ffill()
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].bfill()
day_num = org_df['DayNum'] - org_df[col] + 1
org_df[col_final] =(day_num - day_num.where(day_num<0 ).fillna(0)).fillna(0)
 return col, col_final
db_size = pd.read_sql('SELECT page_count * page_size as size FROM pragma_page_count() , pragma_page_size() ', conn)['size'][0]
print(f'Total size of database is: {db_size/1e9:0.3f} GB' )<categorify>
df_population = pd.read_csv(base_folder + 'worldpopulaton-ver2/all_population.csv', delimiter=';', decimal=',', na_values='N.A.')
 df_population['Urban Pop'] = df_population['Urban Pop'].fillna(100.0)
 df_population = pd.get_dummies(df_population, columns=['Continent'])
 continent_columns = []
for c in df_population.columns:
if 'Continent_' in c:
continent_columns.append(c)
 df_population
def select_state(batch_cols, records):
return f<categorify>
countries_to_replace = [ 
('Czech Republic', 'Czechia'), 
('United States of America', 'US'), 
('Côte d'Ivoire(Ivory Coast)', 'Côte d'Ivoire'), 
('Korea(South)', 'Korea, South'), 
('Swaziland', 'Eswatini'), 
('Myanmar(Burma)', 'Burma'), 
('East Timor', 'Timor-Leste'), 
('Macedonia', 'North Macedonia'), 
('Cape Verde', 'Cabo Verde'), 
('Congo(Republic)', 'Congo(Brazzaville)'), 
('Congo(Democratic Republic)', 'Congo(Kinshasa)'), 
('Palestinian Territories', 'West Bank and Gaza'), 
('United Kingdom of Great Britain and Northern Ireland', 'United Kingdom'), 
('Vatican City', 'Holy See'), 
('Sao Tome & Principe', 'Sao Tome and Principe')
 ]
country_state_pairs_to_replace = [ 
(( 'Greenland', 'entire country'),('Denmark', 'Greenland')) , 
(( 'Anguilla', 'entire country'),('United Kingdom', 'Anguilla')) , 
(( 'Bermuda', 'entire country'),('United Kingdom', 'Bermuda')) , 
(( 'British Virgin Islands', 'entire country'),('United Kingdom', 'British Virgin Islands')) , 
(( 'Isle of Man', 'entire country'),('United Kingdom', 'Isle of Man')) , 
(( 'Turks and Caicos Islands', 'entire country'),('United Kingdom', 'Turks and Caicos Islands')) , 
(( 'Sint Maarten', 'entire country'),('Netherlands', 'Sint Maarten')) , 
(( 'Saint Pierre & Miquelon', 'entire country'),('France', 'Saint Pierre and Miquelon')) , 
(( 'Falkland Islands', 'entire country'),('United Kingdom', 'Falkland Islands(Malvinas)'))
]
csv_dir = base_folder + 'worldbymap-ver2/'
files = [
'labor_force',
'death_rate',
'air_traffic_passengers',
'hospital_bed_density',
'obesity',
'old_people',
'physicians_density',
'cigarettes',
'alcohol'
]
wbm = {}
for f in files:
wbm[f] = pd.read_csv(csv_dir + f + '.csv', delimiter=';', decimal=',', na_values='N.A.')
 for ctr in countries_to_replace:
wbm[f]['Country'] = wbm[f]['Country'].replace(ctr[0], ctr[1])
 for pair in country_state_pairs_to_replace:
ind =(wbm[f]['Country'] == pair[0][0])&(wbm[f]['State'] == pair[0][1])
 wbm[f].loc[ind, 'Country'] = pair[1][0]
wbm[f].loc[ind, 'State'] = pair[1][1]
wbm[files[0]]
def update_state(df):
def get_select_params(r):
values_uc = f'({r.user_id}, {r.content_id}, {r.answered_correctly}, {1-r.answered_correctly})'
values_u = f'({r.user_id}, {r.answered_correctly}, {1-r.answered_correctly})'
return values_uc, values_u
values = df.apply(get_select_params, axis=1, result_type='expand')
 return f<merge>
df_add = pd.DataFrame()
for dataset in wbm.keys() :
if df_add.shape ==(0, 0):
df_add = wbm[dataset].copy()
else:
df_add = pd.merge(df_add, wbm[dataset], on=['Country', 'State'], how='left')
 df_add.rename(columns={"Country": "Country/Region", "State": "Province/State"}, inplace=True)
 df_add
def data_transform(df,FEATURE_COLS, is_training = True, is_validation = True):
df = df.join(Question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df['is_beginning'] = df['user_count'] < 20
df['attemp'] = df.groupby(['user_id','content_id'] ).content_id.transform('cumcount' ).astype(np.uint8)
 df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS1 + TARGET_COL]
else:
df = df[FEATURE_COLS1]
return df
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'])
 prior_df = pd.DataFrame()
current_df = pd.DataFrame()
TARGET_COL = ['answered_correctly']
FEATURE_COLS1 = ['row_id', 'user_count','performance', 'question_average']
FEATURE_COLS2 = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time'] 
<load_from_csv>
df_external = pd.merge(df_population, df_add, on=['Country/Region', 'Province/State'], how='left')
 def fill_missing_percapita_values(dfr, feature_total, feature_percapita):
cond =(dfr[feature_percapita].isna())&(df_external[feature_total].notna())&(df_external['Population'].notna())
 ind = df_external[cond].index
df_external.loc[ind, feature_percapita] = df_external.loc[ind, feature_total] / df_external.loc[ind, 'Population'] * 100.0
fill_missing_percapita_values(df_external, 'LaborForceTotal', 'LaborForcePerCapita')
 fill_missing_percapita_values(df_external, 'AirTrafficPassengersTotal', 'AirTrafficPassengersPerCapita')
 df_external[pop_cols+ext_cols] = df_external[pop_cols+ext_cols].apply(lambda x: x.fillna(x.mean()),axis=0)
 df_external
Question_df = pd.read_csv('.. /input/featuresriid/question.csv')
 user_df = pd.read_csv('.. /input/featuresriid/user.csv' )<define_variables>
df_pop = pd.merge(df, df_external, on=['Country/Region', 'Province/State'], how='left')
 df_pop
model_lgb1 = lgb.Booster(model_file='.. /input/featuresriid/model_twofeatures.txt')
 model_lgb2 = lgb.Booster(model_file='.. /input/7features760/model.txt' )<feature_engineering>
cond_ctry = [ 
(( df['Country/Region']=='Germany')&(df['Province/State']=='entire country'), 'blue'), 
(( df['Country/Region']=='China')&(df['Province/State']=='Hubei'), 'green'), 
(( df['Country/Region']=='Italy')&(df['Province/State']=='entire country'), 'cyan'), 
(( df['Country/Region']=='Spain')&(df['Province/State']=='entire country'), 'magenta'), 
(( df['Country/Region']=='Korea, South')&(df['Province/State']=='entire country'), 'gray'),
]
start_day = 'Day_F25'
cond_day = df[start_day]>0
feature = 'Fatalities'
for i, cnd in enumerate(cond_ctry):
my_df = df_pop[(cnd[0])&(cond_day)]
my_x = my_df[start_day]
my_y = my_df[feature]/my_df['PreviousDay-11ConfirmedCases']
chart_data = pd.DataFrame({
'x': my_x,
'y': my_y})
 sns.lineplot(x='x', y='y', data=chart_data, color=cnd[1] )
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1<feature_engineering>
ccc = pop_cols + ext_cols
df_pop[df_pop[ccc].isnull().any(axis=1)][['Country/Region', 'Province/State'] + ccc].drop_duplicates(subset=['Country/Region', 'Province/State'])
%%time
df_batch_prior = None
counter = 0
iter_nb = 0
TARGET = 'answered_correctly'
for test_batch in iter_test:
counter += 1 
(current_df, sample_prediction_df)= test_batch
if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 Question_df = build_question_df(prior_df,Question_df)
 user_df = build_user_df(prior_df,user_df,Question_df)
 answers = eval(test_batch[0]['prior_group_answers_correct'].iloc[0])
 df_batch_prior['answered_correctly'] = answers
cursor.executescript(update_state(df_batch_prior[df_batch_prior.content_type_id == 0]))
if not counter % 100:
conn.commit()
prior_df = current_df.copy()
current_df = data_transform(current_df,FEATURE_COLS1,False,False)
 predictions0 = model_lgb1.predict(current_df.iloc[:,1:])
 iter_nb = 1
print('--')
(test_df, sample_prediction_df)= test_batch
answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 content_df = pd.read_csv('.. /input/7fetscontent/content.csv')
 qUestion_df = pd.read_csv('.. /input/7fetscontent/question.csv')
 if df_batch_prior is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, qUestion_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(25439.41)
 predictions1 = model_lgb2.predict(test_df[FEATURE_COLS2])
 df_batch_prior = test_batch[0][batch_cols_prior].astype({k: dtypes[k] for k in batch_cols_prior})
 df_batch = test_batch[0][test_batch[0].content_type_id == 0]
records = df_batch[batch_cols].fillna(0 ).to_records(index=False)
 df_batch = pd.read_sql(select_state(batch_cols, records), conn)
 predictions2 = model_xgb.predict(xgb.DMatrix(df_batch[test_cols]))
df_batch['answered_correctly'] = predictions0*0.4+predictions2*0.6
env.predict(df_batch[['row_id', 'answered_correctly']])
<compute_test_metric>
df_pop['ExposedDensity'] =(df_pop['Population'] - np.power(df_pop['ConfirmedCases'], 1.43)) /df_pop['Land Area']
density = df_pop.groupby(['Country/Region', 'Province/State'])['Density']
df_pop['PreviousDay-0ExposedDensity'] = df_pop.groupby(['Country/Region', 'Province/State'])['ExposedDensity'].shift(periods=1, fill_value=np.nan)
 df_pop['PreviousDay-0ExposedDensity'] = df_pop.apply(
 lambda row: row['Density'] if np.isnan(row['PreviousDay-0ExposedDensity'])else row['PreviousDay-0ExposedDensity'],
axis=1 
)
 df_pop
predictions2,predictions1,predictions0<import_modules>
model_x_columns_without_dummies = add_cols + pop_cols + ext_cols + previous_days_cols + special_cols2
model_x_columns = model_x_columns_without_dummies + continent_columns
def train_test_split(X, y, test_size=0.3, random_state=0):
day_first = min(X['DayNum'])
 day_last = max(X['DayNum'])
 number_of_days_for_train = int(round(( day_last-day_first+1)*(1-test_size),0))
last_day_for_training = number_of_days_for_train + day_first - 1
X_tr = X[X['DayNum']<=last_day_for_training].copy()
y_tr = y[X['DayNum']<=last_day_for_training].copy()
X_te = X[X['DayNum']>last_day_for_training].copy()
y_te = y[X['DayNum']>last_day_for_training].copy()
return X_tr, X_te, y_tr, y_te
def rmsle(ytrue, ypred):
return np.sqrt(mean_squared_log_error(ytrue, ypred))
def mae(ytrue, ypred):
return mean_absolute_error(ytrue, ypred)
 def mse(ytrue, ypred):
return mean_squarred_error(ytrue, ypred)
 def analyse3(tr_true, tr_pred, val_true, val_pred, test_true, test_pred):
chart_data0 = pd.DataFrame({
'x0': tr_true.flatten() ,
'x1': tr_pred.flatten() ,
'y': tr_true.flatten() -tr_pred.flatten() })
 chart_data1 = pd.DataFrame({
'x0': val_true.flatten() ,
'x1': val_pred.flatten() ,
'y': val_true.flatten() -val_pred.flatten() })
 chart_data2 = pd.DataFrame({
'x0': test_true.flatten() ,
'x1': test_pred.flatten() ,
'y': test_true.flatten() -test_pred.flatten() })
 fig, ax =plt.subplots(1,3)
 sns.scatterplot(x='x0', y='y', data=chart_data0, color='black', ax=ax[0])
 sns.scatterplot(x='x1', y='y', data=chart_data0, color='red', ax=ax[0])
 sns.scatterplot(x='x0', y='y', data=chart_data1, color='black', ax=ax[1])
 sns.scatterplot(x='x1', y='y', data=chart_data1, color='red', ax=ax[1])
 sns.scatterplot(x='x0', y='y', data=chart_data2, color='black', ax=ax[2])
 sns.scatterplot(x='x1', y='y', data=chart_data2, color='red', ax=ax[2])
 print('MAE train: {}'.format(round(rmsle(tr_true, tr_pred), 6)))
 print('MAE val: {}'.format(round(rmsle(val_true, val_pred), 6)))
 print('MAE test: {}'.format(round(rmsle(test_true, test_pred), 6)))
 def prepare_data(df, what_to_predict, test_size=0.3, dropna=False):
df_tmp = df.copy()
if dropna:
df_tmp.dropna(inplace=True)
 df_tmp.loc[df_tmp[what_to_predict]<0, what_to_predict] = 0
data_X = df_tmp[model_x_columns+['Country/Region']]
data_y = df_tmp[what_to_predict].values.flatten()
return train_test_split(data_X, data_y, test_size=test_size, random_state=42)
 def predict_output(input_data, model):
y_pred = np.abs(model.predict(input_data))
return y_pred
def expm1_relu(y):
tmp = np.expm1(y)
 tmp[tmp<0]=0
return np.around(tmp )
import gc
import psutil
import random
from tqdm.notebook import tqdm
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader<define_variables>
scaler0 = None
scaler1 = None
scaler2 = None
def scale_data(data):
global scaler0, scaler1, scaler2
data_bis = data.copy()
daynum = data_bis['DayNum'].copy()
memory = dict()
for c in special_cols2:
memory[c] = data_bis[c].copy()
if scaler1:
data_bis[model_x_columns_without_dummies] = scaler0.transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler1.transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler2.transform(data_bis[model_x_columns_without_dummies])
 else:
scaler0 = PowerTransformer()
scaler1 = MinMaxScaler()
scaler2 = StandardScaler()
data_bis[model_x_columns_without_dummies] = scaler0.fit_transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler1.fit_transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler2.fit_transform(data_bis[model_x_columns_without_dummies])
 for c in memory.keys() :
data_bis[c] = memory[c]
data_bis['DayNum'] = daynum
return data_bis
df_pop_bis = scale_data(df_pop)
 df_pop_bis
MAX_SEQ = 50
NUM_HEADS = 10
NUM_EMBED = 128
n_skill = 13523<load_from_csv>
model_path = join('.')
 model_file_f = join(model_path, 'nn_model_f.h5')
 model_file_cc = join(model_path, 'nn_model_cc.h5' )
%%time
dtypes = {'timestamp': 'int64',
'user_id': 'int32' ,
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtypes)
<sort_values>
class Swish(Activation):
def __init__(self, activation, **kwargs):
super(Swish, self ).__init__(activation, **kwargs)
 self.__name__ = 'swish'
def swish(x, beta = 0.6):
return(x * sigmoid(beta * x))
get_custom_objects().update({'swish': Swish(swish)} )
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop=True )<groupby>
model_f = Sequential()
model_f.add(Dense(50, input_dim=len(model_x_columns)-1, activation='swish'))
model_f.add(Dropout(0.2))
model_f.add(Dense(15, activation='elu'))
model_f.add(Dropout(0.2))
model_f.add(Dense(1, activation='elu'))
opt_f = Adam(learning_rate=0.001, beta_1=0.94, beta_2=0.99, amsgrad=False)
 model_f.compile(loss='mean_absolute_error', optimizer=opt_f, metrics=['mae'])
 save_best_only_callback_f = ModelCheckpoint(
 filepath=model_file_f,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min',
period=1 
)
 data_X_tr, data_X_rest, data_y_tr, data_y_rest = prepare_data(df_pop_bis, 'FatalitiesDelta', test_size=0.3, dropna=False)
 data_X_val, data_X_test, data_y_val, data_y_test = train_test_split(data_X_rest, data_y_rest, test_size=0.5, random_state=111)
 history = model_f.fit(x=data_X_tr[model_x_columns].drop(columns=['DayNum']), y=data_y_tr,
validation_data=(data_X_val[model_x_columns].drop(columns=['DayNum']), data_y_val),
epochs=170, batch_size=128, verbose=1, callbacks=[save_best_only_callback_f])
 plt.plot(history.history['loss'][0:])
 plt.plot(history.history['val_loss'][0:])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'val'], loc='upper left')
 plt.show()
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda x:(
 x['content_id'].values,
x['answered_correctly'].values))
del train_df
gc.collect()<define_variables>
model_f = load_model(model_file_f)
 tr_pred = predict_output(data_X_tr[model_x_columns].drop(columns=['DayNum']), model_f)
 val_pred = predict_output(data_X_val[model_x_columns].drop(columns=['DayNum']), model_f)
 test_pred = predict_output(data_X_test[model_x_columns].drop(columns=['DayNum']), model_f)
 analyse3(data_y_tr, tr_pred,
data_y_val, val_pred,
data_y_test, test_pred )
random.seed(1 )<define_variables>
model_cc = Sequential()
model_cc.add(Dense(28, input_dim=len(model_x_columns)-1, activation='swish'))
model_cc.add(Dropout(0.0))
model_cc.add(Dense(15, activation='elu'))
model_cc.add(Dropout(0.0))
model_cc.add(Dense(1, activation='swish'))
opt_cc = Adam(learning_rate=0.0001, beta_1=0.988, beta_2=0.99, amsgrad=False)
 model_cc.compile(loss='mean_absolute_error', optimizer=opt_cc, metrics=['mae'])
 save_best_only_callback_cc = ModelCheckpoint(
 filepath=model_file_cc,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min',
period=1 
)
 data_X_tr, data_X_rest, data_y_tr, data_y_rest = prepare_data(df_pop_bis, 'ConfirmedCasesDelta', test_size=0.3, dropna=False)
 data_X_val, data_X_test, data_y_val, data_y_test = train_test_split(data_X_rest, data_y_rest, test_size=0.5, random_state=111)
 history = model_cc.fit(x=data_X_tr[model_x_columns].drop(columns=['DayNum']), y=data_y_tr,
validation_data=(data_X_val[model_x_columns].drop(columns=['DayNum']), data_y_val),
epochs=770, batch_size=128, verbose=1, callbacks=[save_best_only_callback_cc])
 plt.plot(history.history['loss'][0:])
 plt.plot(history.history['val_loss'][0:])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'val'], loc='upper left')
 plt.show()
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label<load_pretrained>
model_cc = load_model(model_file_cc)
 tr_pred = predict_output(data_X_tr[model_x_columns].drop(columns=['DayNum']), model_cc)
 val_pred = predict_output(data_X_val[model_x_columns].drop(columns=['DayNum']), model_cc)
 test_pred = predict_output(data_X_test[model_x_columns].drop(columns=['DayNum']), model_cc)
 analyse3(( data_y_tr),(tr_pred), 
(data_y_val),(val_pred), 
(data_y_test),(test_pred))
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
<define_search_model>
df_test = pd.read_csv(data_base + 'test.csv')
 df_test.rename(columns={'Province_State': 'Province/State', 'Country_Region': 'Country/Region'}, inplace=True)
 df_test['Province/State'].fillna('entire country', inplace=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'] ).dt.date
df_test['DayNum'] =(df_test['Date'] - day_zero ).apply(lambda x: int(x.days))
for c1, c2 in zip(special_cols1, special_cols2):
zero_days = pd.DataFrame(df.groupby(['Country/Region', 'Province/State', c1] ).size().reset_index() [['Country/Region', 'Province/State', c1]])
 zero_days.drop_duplicates(subset=['Country/Region', 'Province/State'], keep='last', inplace=True)
 df_test = df_test.merge(zero_days, on=['Country/Region', 'Province/State'], how='left')
 real_day_num = df_test['DayNum'] - df_test[c1] + 1
df_test[c2] =(real_day_num - real_day_num.where(real_day_num<0 ).fillna(0)).fillna(0)
 df_test
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<choose_model_class>
df_test_pop = pd.merge(df_test, df_external, on=['Country/Region', 'Province/State'], how='left')
 df_test_pop
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device )<train_model>
output_columns = ['ConfirmedCases', 'Fatalities']
tmp_output_columns = ['ConfirmedCases_y', 'Fatalities_y']
last_training_day = df['DayNum'].max()
first_test_day = df_test['DayNum'].min()
train_test_keys = ['Country/Region', 'Province/State', 'DayNum']
df_test_pop_train = pd.merge(df_test_pop, df_pop[df_pop['DayNum']>=first_test_day][train_test_keys + ['PreviousDay-0ExposedDensity', 'ExposedDensity'] + previous_days_cols + output_columns],
on=train_test_keys, how='left')
 df_test_pop_train
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc<train_model>
df_test_final = pd.concat([df_pop[df_pop['DayNum']<first_test_day], df_test_pop_train] ).reset_index(drop=True)
 df_test_final[(df_test_final['Country/Region']=='Poland')&(df_test_final['DayNum']<=last_training_day+1)&(df_test_final['DayNum']>last_training_day-10)]
epochs = 35
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))<save_model>
df_test_final['PopulationOrg'] = df_test_final['Population'].copy()
df_test_final['Land Area Org'] = df_test_final['Land Area'].copy()
torch.save(model.state_dict() , "SAKT.pt")
 del dataset
gc.collect()<split>
model_cc = load_model(model_file_cc)
 model_f = load_model(model_file_f)
 last_test_day = df_test['DayNum'].max()
for day in range(last_training_day+1, last_test_day+1):
print('predicting day {}({} to go)'.format(day, last_test_day-day))
add_extra_features_from_previous_days(df_test_final, TAIL)
 df_test_final['PreviousDay-0ConfirmedCases_NotScaled'] = df_test_final['PreviousDay-0ConfirmedCases'].copy()
df_test_final['PreviousDay-0Fatalities_NotScaled'] = df_test_final['PreviousDay-0Fatalities'].copy()
df_test_final['PreviousDay-0ExposedDensity'] = df_test_final.groupby(['Country/Region', 'Province/State'])['ExposedDensity'].shift(periods=1, fill_value=np.nan)
 df_test_final['PreviousDay-0ExposedDensity'] = df_test_final.apply(
 lambda row: row['Density'] if np.isnan(row['PreviousDay-0ExposedDensity'])else row['PreviousDay-0ExposedDensity'],
axis=1 
)
 current_day = df_test_final[df_test_final['DayNum']==day].copy()
current_day_scaled = scale_data(current_day)
 current_day_scaled['ConfirmedCasesDelta'] = predict_output(current_day_scaled[model_x_columns].drop(columns=['DayNum']), model_cc)
 current_day_scaled['FatalitiesDelta'] = predict_output(current_day_scaled[model_x_columns].drop(columns=['DayNum']), model_f)
 current_day_scaled['ConfirmedCases'] = current_day_scaled['PreviousDay-0ConfirmedCases_NotScaled'] + current_day_scaled['ConfirmedCasesDelta']
current_day_scaled['Fatalities'] = current_day_scaled['PreviousDay-0Fatalities_NotScaled'] + current_day_scaled['FatalitiesDelta']
current_day_scaled['ExposedDensity'] =(current_day_scaled['PopulationOrg'] - np.power(current_day_scaled['ConfirmedCases'], 1.43)) /current_day_scaled['Land Area Org']
cond = df_test_final['DayNum']==day
df_test_final.loc[cond, output_columns+['ExposedDensity']] = current_day_scaled[output_columns+['ExposedDensity']].copy()
df_test_final.loc[(cond)&(df_test_final[cond]['ConfirmedCases']<current_day_scaled['ConfirmedCases']), 'ConfirmedCases'] = df_test_final['PreviousDay-0ConfirmedCases']
df_test_final.loc[(cond)&(df_test_final[cond]['Fatalities']<current_day_scaled['Fatalities']), 'Fatalities'] = df_test_final['PreviousDay-0Fatalities']
df_test_final.columns
env = riiideducation.make_env()
iter_test = env.iter_test() 
<find_best_params>
submission_columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']
df_test_final.loc[df_test_final['ForecastId'].isna() , 'ForecastId'] = 0
df_test_final[submission_columns] = df_test_final[submission_columns].astype(int)
 df_test_final[df_test_final['DayNum']>=first_test_day][submission_columns].to_csv('submission.csv', index=False)
 df_test_final[df_test_final['DayNum']>=last_training_day][submission_columns]