Kaggle competition: COVID19 Global Forecasting (Week 4)
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
N = 5
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<define_variables>
def get_preds_lin_reg(series, pred_min, H):

 regr = LinearRegression(fit_intercept=True)
 pred_list = []
X_train = np.array(range(len(series)))
 y_train = np.array(series)
 X_train = X_train.reshape(-1, 1)
 y_train = y_train.reshape(-1, 1)
 regr.fit(X_train, y_train)
 pred = regr.predict(np.array(range(len(series),len(series)+H)).reshape(-1,1))
pred = pred.reshape(H,)
 pred[pred < pred_min] = pred_min
return np.around(pred )
SCALING = 5<define_variables>
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 train.columns = [col.lower() for col in train.columns]
train['date'] = pd.to_datetime(train['date'], format='%Y-%m-%d')
 train
random.seed(42)
 TQDM_INT = 8
EPOCHS = 10
MAX_SEQ = 150
NUM_EMBED = 128
NUM_HEADS = 8
WORKERS = 4
BATCH_SIZE = 1024
VAL_BATCH_SIZE = 4096
n_skill = 13523
LEARNING_RATE = 1e-3<define_variables>
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test.columns = [col.lower() for col in test.columns]
test['date'] = pd.to_datetime(test['date'], format='%Y-%m-%d')
 test
TRAIN_DTYPES = {'timestamp':'int64',
'user_id':'int32' ,
'content_id':'int16',
'content_type_id':'int8',
'answered_correctly':'int8'}
TRAIN_COLS = TRAIN_DTYPES.keys()<load_from_csv>
submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 submission
%%time
train_df = pd.read_parquet('.. /input/cv-strategy-in-the-kaggle-environment/cv3_train.parquet')
 train_df = train_df[TRAIN_COLS].astype(TRAIN_DTYPES)
 train_df = train_df[train_df["content_type_id"] == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
<groupby>
train.isnull().sum(axis=0 )
SMOOTHING_FACTOR=0.2
train_df[['answered_correctly']] = train_df[['answered_correctly']]*(1-SMOOTHING_FACTOR)\
+ SMOOTHING_FACTOR/2
train_group = train_df[['user_id', 'content_id', 'answered_correctly']]\ 
.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))<set_options>
print(len(train['province_state'].unique()))
train['province_state'].unique()
del train_df
gc.collect() ;<data_type_conversions>
print(len(train['country_region'].unique()))
train['country_region'].unique()
valid_df = pd.read_parquet('.. /input/cv-strategy-in-the-kaggle-environment/cv3_valid.parquet')
 valid_df = valid_df[TRAIN_COLS].astype(TRAIN_DTYPES)
 valid_df = valid_df[valid_df["content_type_id"] == False]
valid_group = valid_df[['user_id', 'content_id', 'answered_correctly']]\ 
.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
del valid_df
gc.collect() ;<define_search_model>
train['country_region'].value_counts()
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill,
max_seq=MAX_SEQ,
embed_dim=NUM_EMBED,
num_heads=NUM_HEADS,
num_layers=1):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x+att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, subset="train", max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.subset = subset
self.user_ids = []
for user_id in group.index:

 q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=np.float16)
 if seq_len >= self.max_seq:
if self.subset == "train":
if random.random() > 0.1:
random_start_index = random.randint(0, seq_len - self.max_seq)

end_index = random_start_index + self.max_seq
q[:] = q_[random_start_index:end_index]
qa[:] = qa_[random_start_index:end_index]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
seq_len = random.randint(2,seq_len)
 q[-seq_len:] = q_[:seq_len]
qa[-seq_len:] = qa_[:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label
def train_epoch(model, train_iterator, optim, criterion, device="cuda"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
len_dataset = len(train_iterator)
 for idx, item in enumerate(train_iterator):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label =(label[:, -1] >=0.5 ).long()
output = torch.sigmoid(output)
 pred =(output >= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc
def valid_epoch(model, valid_iterator, criterion, device="cuda", scaling=1):
model.eval()
valid_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
len_dataset = len(valid_iterator)
 for idx, item in enumerate(valid_iterator):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
with torch.no_grad() :
output, _ = model(x, target_id)
 loss = criterion(output, label)
 valid_loss.append(loss.item())
 output = scaling*output[:, -1]
output = torch.sigmoid(output)
 label = label[:, -1]
pred =(output >= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(valid_loss)
 return loss, acc, auc<create_dataframe>
train[train['country_region']=='Singapore']
train_dataset = SAKTDataset(train_group, n_skill, subset="train")
 train_loader = DataLoader(train_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=WORKERS)
 valid_dataset = SAKTDataset(valid_group, n_skill, subset="valid")
 val_loader = DataLoader(valid_dataset,
batch_size=VAL_BATCH_SIZE,
shuffle=False,
num_workers=WORKERS )<prepare_x_and_y>
train['province_state'] = train['province_state'].fillna(value = 'nil')
 train.head()
item = train_dataset.__getitem__(5)
 print("x", len(item[0]), item[0], '
')
 print("target_id", len(item[1]), item[1] , '
')
 print("label", len(item[2]), item[2], '
' )<choose_model_class>
test['province_state'] = test['province_state'].fillna(value = 'nil')
 test.head()
def get_num_params(model):
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
 no_params = sum([np.prod(p.size())for p in model_parameters])
 return no_params
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=NUM_EMBED, num_heads=NUM_HEADS)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device)
 num_params = get_num_params(model)
 print(f"
print(f"
print(f"seq len : {MAX_SEQ}")
 print(f"<train_model>
ps_cr_unique = train[['province_state', 'country_region']].drop_duplicates()
ps_cr_unique
epochs = 10
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, train_loader, optimizer, criterion, device)
 print(f"Epoch - [{epoch}/{epochs}]")
 print(f"Train with label smoothing: loss - {loss:.4f} acc - {acc:.4f} auc - {auc:.4f}")
 val_loss, val_acc, val_auc = valid_epoch(model, val_loader, criterion, device=device)
 print(f"Valid without scaling : loss - {val_loss:.4f} acc - {val_acc:.4f} auc - {val_auc:.4f}")
 val_loss, val_acc, val_auc = valid_epoch(model, val_loader, criterion, device=device, scaling=SCALING)
 print(f"Valid with a scaling of {SCALING} : loss - {val_loss:.4f} acc - {val_acc:.4f} auc - {val_auc:.4f}" )<define_variables>
date_max_train = train[(train['province_state']=='nil')& 
(train['country_region']=='Singapore')]['date'].max()
date_max_test = test[(test['province_state']=='nil')& 
(test['country_region']=='Singapore')]['date'].max()
pred_days =(date_max_test - date_max_train ).days
print(date_max_train, date_max_test, pred_days )
SCALING = 3.5
model_file = '.. /input/riiid-models/sakt_layer_1_head_8_embed_128_seq_150_auc_0.7605.pt'<split>
ps = 'nil'
cr = 'Singapore'
env = riiideducation.make_env()
iter_test = env.iter_test()<data_type_conversions>
train_sgp = train[(train['province_state']==ps)&(train['country_region']==cr)]
train_sgp[-5:]
train_df = pd.read_parquet('.. /input/cv-strategy-in-the-kaggle-environment/cv3_train.parquet')
 train_df = train_df[TRAIN_COLS].astype(TRAIN_DTYPES)
 train_df = train_df[train_df["content_type_id"] == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
 group = train_df[['user_id', 'content_id', 'answered_correctly']]\ 
.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))<predict_on_test>
preds = get_preds_lin_reg(train_sgp['confirmedcases'][-N:], 0, pred_days)
 preds
%%time
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(prev_test_df is not None)&(psutil.virtual_memory().percent<95):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill)
 test_dataloader = DataLoader(test_dataset, batch_size=25600, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = model(x, target_id)
 output = torch.sigmoid(SCALING*output)
 output = output[:, -1]
output = 0.25 + 0.75*output
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0,
['row_id', 'answered_correctly']] )<load_from_csv>
date_list = []
date = pd.date_range(date_max_train+timedelta(days=1), date_max_test)
 results = pd.DataFrame({'date': date, 'preds':preds})
 results.head()
sns.set()
sub = pd.read_csv('.. /working/submission.csv')
 sub['answered_correctly'].hist() ;<feature_engineering>
test_merged = test.merge(train[['province_state', 'country_region', 'date', 'confirmedcases', 'fatalities']],
left_on=['province_state', 'country_region', 'date'],
right_on=['province_state', 'country_region', 'date'],
how='left')
 test_merged
<set_options>
test_merged2 = test_merged.merge(results,
left_on=['province_state', 'country_region', 'date'],
right_on=['province_state', 'country_region', 'date'],
how='left')
 test_merged2
def reduce_mem_usage(df, use_float16=False,verbose=True):

 start_mem = df.memory_usage().sum() / 1024**2
if verbose :print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
if verbose:print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
if verbose:print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df<import_modules>
test_merged2['confirmedcases'] = test_merged2.apply(lambda row: row['confirmedcases_x'] if pd.isnull(row['confirmedcases_y'])else row['confirmedcases_y'], axis=1)
 test_merged2.drop(['confirmedcases_x', 'confirmedcases_y'], axis=1, inplace=True)
 test_merged2
import random
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import gc
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler,MinMaxScaler,PolynomialFeatures
import json
from sklearn.metrics import roc_auc_score,accuracy_score
import lightgbm as lgb
import dill
import time
from sklearn.pipeline import Pipeline,FeatureUnion
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest, chi2, f_regression
from sklearn.model_selection import train_test_split
from sklearn.base import BaseEstimator, ClassifierMixin
import xgboost as xgb
import tensorflow as tf
from tensorflow import keras<define_variables>
test_merged3 = test_merged2.merge(results,
left_on=['province_state', 'country_region', 'date'],
right_on=['province_state', 'country_region', 'date'],
how='left')
 test_merged3
np.random.seed(59)
 tf.random.set_seed(59)
 random.seed(59)
 path_train1 = '.. /input/riiid-preprocess-and-balance-the-dataset/train0.feather'
path_train2 = '.. /input/riiid-preprocess-and-balance-the-dataset/train1.feather'
path_content = '.. /input/riiid-preprocess-and-balance-the-dataset/content.feather'
path_metadata = '.. /input/riiid-preprocess-and-balance-the-dataset/metadata.dill'<define_variables>
test_merged3['fatalities'] = test_merged3.apply(lambda row: row['fatalities_x'] if pd.isnull(row['fatalities_y'])else row['fatalities_y'], axis=1)
 test_merged3.drop(['fatalities_x', 'fatalities_y'], axis=1, inplace=True)
 test_merged3
SUBMIT = True<load_from_disk>
submission = test_merged3.copy()
submission.drop(['country_region', 'province_state', 'date'], axis=1, inplace=True)
 submission.rename(columns={'forecastid': 'ForecastId',
'fatalities': 'Fatalities',
'confirmedcases': 'ConfirmedCases'}, inplace=True)
 submission
%%skip SUBMIT
metadata = dill.load(open(path_metadata,'rb'))
target = 'answered_correctly'
train_df = pd.read_feather(path_train1)
 train_df = train_df.append(pd.read_feather(path_train2))
train_df.shape<merge>
submission.to_csv("submission.csv", index=False )