Kaggle competition: COVID19 Global Forecasting (Week 4)
model.eval()
prev_test_df = None<feature_engineering>
warnings.filterwarnings("ignore" )
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent < 90):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id',
'content_id',
'answered_correctly']]\ 
.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0],
prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1],
prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0,
['row_id', 'answered_correctly']] )<import_modules>
df_train=pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_test=pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 df_sub=pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(df_train.shape)
 print(df_test.shape)
 print(df_sub.shape )
import pickle
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict, deque
from tqdm.notebook import tqdm
import lightgbm as lgb<define_variables>
print(f"Unique Countries: {len(df_train.Country_Region.unique())}" )
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'<categorify>
train_dates=list(df_train.Date.unique())
 latest_date=df_train.Date.max()
print(f"Period : {len(df_train.Date.unique())} days")
 print(f"From : {df_train.Date.min() } To : {df_train.Date.max() }" )
last_time_u_dict = defaultdict(int)
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 past_question = defaultdict(list)
 past_time_diff = defaultdict(list)
 past_prior_elaps = defaultdict(list)
 past_answer = defaultdict(list )<feature_engineering>
print(f"Unique Regions: {df_train.shape[0]/len(df_train.Date.unique())}" )
def add_user_feats(df,last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 td = np.zeros(len(df), dtype=np.int64)
 for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly','timestamp']].values)) :
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
td[cnt] = row[2] - last_time_u_dict[row[0]]
if row[2] == last_time_u_dict[row[0]]:
td[cnt] = td[cnt-1]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
last_time_u_dict[row[0]] = row[2]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu, 'time_diff':td})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df 
<categorify>
df_train.Country_Region.value_counts()
def add_user_feats2(df, past_question, past_answer, past_prior_elaps, past_time_diff):
for _, row in enumerate(tqdm(df[['user_id','answered_correctly','content_id','prior_question_elapsed_time','time_diff']].values)) :
if row[0] not in past_question:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[2])
 past_question[row[0]] = new_list
else:
past_question[row[0]].append(row[2])
 if row[0] not in past_answer:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[1])
 past_answer[row[0]] = new_list
else:
past_answer[row[0]].append(row[1])
 if row[0] not in past_prior_elaps:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[3])
 past_prior_elaps[row[0]] = new_list
else:
past_prior_elaps[row[0]].append(row[3])
 if row[0] not in past_time_diff:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[4])
 past_time_diff[row[0]] = new_list
else:
past_time_diff[row[0]].append(row[4] )<data_type_conversions>
print(f"Number of rows without Country_Region : {df_train.Country_Region.isna().sum() }" )
def add_user_feats_without_update(df ,last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 td = np.zeros(len(df), dtype=np.int64)
 for cnt, row in enumerate(df[['user_id','part','timestamp']].values):
td[cnt] = row[2] - last_time_u_dict[row[0]]
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu, 'time_diff':td})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df<categorify>
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True)
 df_train
def get_user_feats_for_nn_without_update(df, past_question, past_answer, past_prior_elaps, past_time_diff):
current_question = []
past_question_answer = []
past_answer_correctly = []
past_time = []
past_prior = []
for cnt,row in enumerate(tqdm(df[['user_id','content_id','prior_question_elapsed_time','time_diff']].values)) :
if row[0] not in past_answer:
temp_answer = [0]*30
temp_past_answer = [0]*30
else:
temp_answer = past_answer[row[0]].copy()
temp_past_answer = past_answer[row[0]].copy()
if row[0] not in past_question:
temp_question = [0]*30
else:
temp_question = past_question[row[0]].copy()
temp_past_answer= [x+1 if y > 0 else 0 for x , y in zip(temp_past_answer, temp_question)]
past_answer_correctly.append(temp_past_answer)
 temp_past_question_answer = [x+y*13523 for x,y in zip(temp_question, temp_answer)]
past_question_answer.append(temp_past_question_answer)
 temp_question.append(row[1]+1)
 current_question.append([temp_question[i] for i in range(30)])
 if row[0] not in past_prior_elaps:
temp_elaps = [0]*29 + [row[2]/3e5]
else:
temp_elaps = past_prior_elaps[row[0]].copy()
temp_elaps.append(row[2]/3e5)
 past_prior.append(temp_elaps)
 if row[0] not in past_time_diff:
temp_time_diff = [0]*29 + [row[3]/1e6]
else:
temp_time_diff = past_time_diff[row[0]].copy()
temp_time_diff.append(row[3]/1e6)
 past_time.append(temp_time_diff)
 current_question = np.array(current_question)
 past_question_answer = np.array(past_question_answer)
 past_other_feats = np.dstack(( past_prior,past_time))
past_answer_correctly = np.array(past_answer_correctly)
 return current_question, past_question_answer, past_other_feats, past_answer_correctly<feature_engineering>
df_train["Delta"]=1.0
df_train["NewCases"]=0.0
final_train=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","NewCases","UniqueRegion", "Delta"])
 for region in region_list:
df_temp=df_train[df_train.UniqueRegion==region].reset_index()
size_train=df_temp.shape[0]
df_temp.NewCases[0]=df_temp.ConfirmedCases[1]
for i in range(1,df_temp.shape[0]):
df_temp.NewCases[i]=df_temp.ConfirmedCases[i]-df_temp.ConfirmedCases[i-1]
if(df_temp.ConfirmedCases[i-1]>0):
df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
df_temp=df_temp[["Date","ConfirmedCases","Fatalities","UniqueRegion","NewCases","Delta"]]
final_train=pd.concat([final_train,df_temp], ignore_index=True)
 final_train.shape
def update_user_feats(df, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id', 'answered_correctly', 'content_type_id', 'timestamp', 'content_id','prior_question_elapsed_time','time_diff']].values:
if row[2] == 0:
if row[0] not in past_question:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[4]+1)
 past_question[row[0]] = new_list
else:
past_question[row[0]].append(row[4]+1)
 if row[0] not in past_answer:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[1])
 past_answer[row[0]] = new_list
else:
past_answer[row[0]].append(row[1])
 if row[0] not in past_prior_elaps:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[5]/3e5)
 past_prior_elaps[row[0]] = new_list
else:
past_prior_elaps[row[0]].append(row[5]/3e5)
 if row[0] not in past_time_diff:
new_list = deque([0]*30, maxlen = 30)
 if row[6] >= 1e6:
new_list.append(0)
 else:
new_list.append(row[6]/1e6)
 past_time_diff[row[0]] = new_list
else:
if row[6] >= 1e6:
past_time_diff[row[0]].append(0)
 else:
past_time_diff[row[0]].append(row[6]/1e6)
 last_time_u_dict[row[0]] = row[3]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1<load_pretrained>
latest_train=final_train[final_train.Date==latest_date]
latest_train.head()
feld_needed = ['user_id','content_id','answered_correctly','timestamp','prior_question_elapsed_time','prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]<filter>
%%time
reg_score_list=[]
period=[]
reg=[]
for n in range(3,10):
for region in region_list:
df_temp=final_train[final_train.UniqueRegion==region]
df_temp=df_temp.tail(n ).reset_index()
date=np.arange(1,n+1)
 model=LinearRegression()
X=date.reshape(-1,1)
 Y=df_temp.Delta
model.fit(X,Y)
 reg.append(region)
 reg_score_list.append(model.score(X,Y))
period.append(n)
 score_df=pd.DataFrame({"Region":reg,"Score":reg_score_list, "Period":period})
 for n in range(3,10):
print(f"Average R2 score for {n} days period :{score_df[score_df.Period==n].Score.mean() }")
 plt.figure(figsize=(10,6))
plt.title(f"Distribution of R2 score for {n} days")
 sns.distplot(score_df[score_df.Period==n].Score)
 plt.show()
train = train.loc[train.answered_correctly != -1].reset_index(drop=True)
 valid = valid.loc[valid.answered_correctly != -1].reset_index(drop=True)
 _=gc.collect()<rename_columns>
print(f"Unique Countries: {len(df_test.Country_Region.unique())}")
 test_dates=list(df_test.Date.unique())
 size_test=len(df_test.Date.unique())
 print(f"Period : {len(df_test.Date.unique())} days")
 print(f"From : {df_test.Date.min() } To : {df_test.Date.max() }")
 print(f"Unique Regions: {df_test.shape[0]/len(df_test.Date.unique())}" )
content_df = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean','std'] ).reset_index()
content_df.columns = ['content_id', 'answered_correctly_avg_c','answered_correctly_std_c']<feature_engineering>
df_test["UniqueRegion"]=df_test.Country_Region
df_test.UniqueRegion[df_test.Province_State.isna() ==False]=df_test.Province_State+" , "+df_test.Country_Region
train = add_user_feats(train, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 valid = add_user_feats(valid, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict )<load_from_csv>
df_test.drop(labels=["ForecastId","Province_State","Country_Region"], axis=1, inplace=True)
 df_test["ConfirmedCases"]=0
df_test["Fatalities"]=0
df_test["NewCases"]=0
df_test["Delta"]=0
questions_df = pd.read_csv(question_file)
 questions_df.tags.fillna('-1-1', inplace = True)
 questions_df['tag'] = pd.factorize(questions_df.tags)[0]<merge>
df_pred=pd.DataFrame(columns=["ConfirmedCases","Fatalities"])
 df_traintest=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion","NewCases","Delta"])
 for region in region_list:
df_temp=final_train[final_train.UniqueRegion==region].reset_index()
n=int(best_n_df[best_n_df.Region==region].N.sum())
 delta_list=np.array(df_temp.tail(n ).Delta ).reshape(-1,1)
 death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
scaler=MinMaxScaler()
X=np.arange(1,n+1 ).reshape(-1,1)
 Y=scaler.fit_transform(delta_list)
 model=LinearRegression()
model.fit(X,Y)
 df_test_app=app_test[app_test.UniqueRegion==region]
df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
df_temp.Delta[i]=max(1,scaler.inverse_transform(model.predict(np.array([n] ).reshape(-1,1))))
df_temp.ConfirmedCases[i]=round(df_temp.ConfirmedCases[i-1]*df_temp.Delta[i],0)
 df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 df_temp.NewCases[i]=df_temp.ConfirmedCases[i]-df_temp.ConfirmedCases[i-1]
df_traintest=pd.concat([df_traintest,df_temp],ignore_index=True)
 df_temp=df_temp.iloc[-size_test:,:]
df_temp=df_temp[["ConfirmedCases","Fatalities"]]
df_pred=pd.concat([df_pred,df_temp], ignore_index=True)
last_time_u_dict = defaultdict(int)
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train1 = add_user_feats(train1, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 valid1 = add_user_feats(valid1, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 train1 = pd.merge(train1, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid1 = pd.merge(valid1, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 train1.time_diff[train1.time_diff >= 1e6] = 1e6
valid1.time_diff[valid1.time_diff >= 1e6] = 1e6
train1.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean, inplace = True)
 valid1.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean, inplace = True)
 add_user_feats2(train1, past_question, past_answer, past_prior_elaps, past_time_diff)
 add_user_feats2(valid1, past_question, past_answer, past_prior_elaps, past_time_diff)
 del(train1)
 del(valid1)
 _=gc.collect()<merge>
df_pred=pd.DataFrame(columns=["ConfirmedCases","Fatalities"])
 df_traintest=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion","NewCases","Delta"])
 for region in region_list:
df_temp=final_train[final_train.UniqueRegion==region].reset_index()
n=10
NewCasesList=df_temp.tail(n ).NewCases
death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
X=np.arange(1,n+1 ).reshape(-1,1)
 Y=NewCasesList
model=LinearRegression()
model.fit(X,Y)
 df_test_app=app_test[app_test.UniqueRegion==region]
df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
df_temp.NewCases[i]=round(max(0,model.predict(np.array([n] ).reshape(-1,1)) [0]),0)
 df_temp.ConfirmedCases[i]=df_temp.ConfirmedCases[i-1]+df_temp.NewCases[i]
df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
df_traintest=pd.concat([df_traintest,df_temp],ignore_index=True)
 df_temp=df_temp.iloc[-size_test:,:]
df_temp=df_temp[["ConfirmedCases","Fatalities"]]
df_pred=pd.concat([df_pred,df_temp], ignore_index=True)
 df_pred.shape
train = pd.merge(train, content_df, on=['content_id'], how="left")
 valid = pd.merge(valid, content_df, on=['content_id'], how="left" )<merge>
df_pred=pd.DataFrame(columns=["ConfirmedCases","Fatalities"])
 df_traintest=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion","NewCases","Delta"])
 for region in region_list:
df_temp=final_train[final_train.UniqueRegion==region].reset_index()
n=7
ConfirmedCasesList=df_temp.tail(n ).ConfirmedCases
death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
polynom=PolynomialFeatures(degree=2)
 X=polynom.fit_transform(np.arange(1,n+1 ).reshape(-1,1))
Y=ConfirmedCasesList
model=LinearRegression()
model.fit(X,Y)
 df_test_app=app_test[app_test.UniqueRegion==region]
df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
pred=round(model.predict(polynom.fit_transform(np.array(n ).reshape(-1,1)))[0],0)
 df_temp.ConfirmedCases[i]=max(df_temp.ConfirmedCases[i-1],pred)
 df_temp.NewCases[i]=df_temp.ConfirmedCases[i]+df_temp.ConfirmedCases[i-1]
df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
df_traintest=pd.concat([df_traintest,df_temp],ignore_index=True)
 df_temp=df_temp.iloc[-size_test:,:]
df_temp=df_temp[["ConfirmedCases","Fatalities"]]
df_pred=pd.concat([df_pred,df_temp], ignore_index=True)
 df_pred.shape
train = pd.merge(train, questions_df[['question_id', 'part','tag']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part','tag']], left_on = 'content_id', right_on = 'question_id', how = 'left' )<data_type_conversions>
df_sub.ConfirmedCases=df_pred.ConfirmedCases
df_sub.Fatalities=df_pred.Fatalities