Kaggle competition: COVID19 Global Forecasting (Week 4)
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = len(skills)
 self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 return x, questions
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SAKT_model = SAKTModel(n_skill, embed_dim=128)
 try:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt"))
except:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt", map_location='cpu'))
SAKT_model.to(device)
 SAKT_model.eval()
group = pickle.load(open("group.pkl", "rb"))
print(psutil.virtual_memory().percent)
<split>
def read_data() :
data = pd.read_csv(".. /input/datatrain4/train4.csv")
 data = data.values
return data
iter_test = env.iter_test()
prior_test_df = None<data_type_conversions>
data = read_data()
%%time
for(test_df, sample_prediction_df)in iter_test:
if(prior_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
lgbm_final = sub_preds / len(clfs)
 test_df[target] = np.array(SAKT_outs)* 0.5 + lgbm_final * 0.5
env.predict(test_df[['row_id', target]] )<install_modules>
def handle_country_text(data):
stats = list(np.unique(data[:, 2]))
for idx, d in enumerate(data):
country = d[2]
id = stats.index(country)
 d[2] = id
return stats, data
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
stats, data = handle_country_text(data )
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
def create_sequences(data, stats):
sequences = []
to_compute = []
for idx, s in enumerate(stats):
seq = data[data[:, 2] == idx]
if pd.isnull(seq[0, 1]):
seq = np.delete(seq, [1], 1)
 else:
to_compute.append(seq)
 stats_p = list(np.unique(seq[:, 1]))
for idx2, s2 in enumerate(stats_p):
seqs2 = seq[seq[:, 1] == s2]
seqs2 = np.delete(seqs2, [0, 1, 3], 1)
 for idx, value in enumerate(reversed(seqs2[:, 1:])) :
if idx + 1 < len(seqs2):
cases = value[0] - seqs2[-(idx + 2), 1]
deaths = value[1] - seqs2[-(idx + 2), 2]
seqs2[-(idx + 1), 1] = cases
seqs2[-(idx + 1), 2] = deaths
offset = float(idx2)/ 10
seqs2[:, 0] = seqs2[:, 0] + offset
sequences.append(seqs2)
 continue
seq = np.delete(seq, [0, 2], 1)
 for idx,value in enumerate(reversed(seq[:,1:])) :
if idx + 1 < len(seq):
cases = value[0] - seq[-(idx + 2), 1]
deaths = value[1] - seq[-(idx + 2), 2]
seq[-(idx + 1), 1] = cases
seq[-(idx + 1), 2] = deaths
sequences.append(seq)
 return np.array(sequences )
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
sequences = create_sequences(data, stats)
 sequences = np.array(sequences)
 sequences_train = np.delete(sequences, [0], 2)
 sequences_train = np.array(sequences_train )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<load_from_csv>
def dain(input):
n_features = 2
mean = Lambda(lambda x: K.mean(input, axis=1))(input)
 adaptive_avg = Dense(n_features,
kernel_initializer=Identity(gain=1.0),
bias=False )(mean)
 adaptive_avg = Reshape(( 1, n_features))(adaptive_avg)
 X = Lambda(lambda inputs: inputs[0] - inputs[1] )([input, adaptive_avg])
 std = Lambda(lambda x: K.mean(x**2, axis=1))(X)
 std = Lambda(lambda x: K.sqrt(x+1e-8))(std)
 adaptive_std = Dense(n_features,
bias=False )(std)
 adaptive_std = Reshape(( 1, n_features))(adaptive_std)
 X = Lambda(lambda inputs: inputs[0] / inputs[1] )([X, adaptive_std])
 avg = Lambda(lambda x: K.mean(x, axis=1))(X)
 gate = Dense(n_features,
activation="sigmoid",
kernel_initializer=Identity(gain=1.0),
bias=False )(avg)
 gate = Reshape(( 1, n_features))(gate)
 X = Lambda(lambda inputs: inputs[0] * inputs[1] )([X, gate])
 return X, adaptive_avg, adaptive_std
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
def build_generator(encoder_input_shape, missing_len, verbose=True):
learning_rate = 0.0002
optimizer = Adam(lr=learning_rate)
 generator_decoder_type ='seq2seq'
encoder_inputs = Input(shape=encoder_input_shape)
 hidden, avg, std = dain(encoder_inputs)
 decoder_outputs = []
encoder = LSTM(128, return_sequences=True, return_state=True)
 lstm_outputs, state_h, state_c = encoder(hidden)
 if generator_decoder_type == 'seq2seq':
states = [state_h, state_c]
decoder_lstm = LSTM(128, return_sequences=True, return_state=True)
 decoder_cases = Dense(1, activation='relu')
 decoder_deaths = Dense(1, activation='relu')
 all_outputs_c = []
all_outputs_d = []
inputs = lstm_outputs
for idx in range(missing_len):
outputs, state_h, state_c = decoder_lstm(inputs, initial_state=states)
 inputs = outputs
outputs = BatchNormalization()(outputs)
 outputs = Flatten()(outputs)
 outputs_cases = decoder_cases(outputs)
 outputs_deaths = decoder_deaths(outputs)
 states = [state_h, state_c]
std_c = Lambda(lambda inputs: inputs[:, 0, 0] )(std)
 avg_c = Lambda(lambda inputs: inputs[:, 0, 0] )(avg)
 outputs_cases = Multiply()([outputs_cases, std_c])
 outputs_cases = Add()([outputs_cases, avg_c])
 std_d = Lambda(lambda inputs: inputs[:, 0, 1] )(std)
 avg_d = Lambda(lambda inputs: inputs[:, 0, 1] )(avg)
 outputs_deaths = Multiply()([outputs_deaths, std_d])
 outputs_deaths = Add()([outputs_deaths, avg_d])
 all_outputs_c.append(outputs_cases)
 all_outputs_d.append(outputs_deaths)
 decoder_outputs_c = Lambda(lambda x: x )(outputs_cases)
 decoder_outputs_d = Lambda(lambda x: x )(outputs_deaths)
 model = Model(inputs=encoder_inputs,
outputs=[decoder_outputs_c, decoder_outputs_d])
 if verbose:
print('
Generator summary: ')
 print(model.summary())
 model.compile(loss='mean_squared_logarithmic_error', optimizer=optimizer)
 return model
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
def backtest2(sequences, model, given, missing):
sequences_test = sequences[:, -given:]
pred_d = []
pred_c = []
for i in range(0, missing):
predictions = model.predict(sequences_test[:, :, ])
 predictions[0][predictions[0] < 0] = 0
predictions[1][predictions[1] < 0] = 0
predictions[1] = np.around(predictions[1].astype(np.double))
predictions[0] = np.around(predictions[0].astype(np.double))
pred = np.concatenate(
 [np.expand_dims(predictions[0], axis=2),
np.expand_dims(predictions[1], axis=2)],
axis=2)
 pred_c.append(pred)
 pred_d.append(predictions[1])
 sequences_test = np.concatenate([sequences_test[:, 1:],
pred],
axis=1)
 predictions = np.array(pred_c[0])
 for i in range(1, len(pred_c)) :
predictions = np.concatenate([predictions,
pred_c[i]],
axis=1)
 seq_cases = sequences[:, :, 0]
seq_death = sequences[:, :, 1]
death = np.cumsum(seq_death, axis=1)
 cases = np.cumsum(seq_cases, axis=1)
 cases = np.around(cases.astype(np.double))
cases[cases < 0] = 0
cases_csv = np.expand_dims(cases[:, -1], axis=1)
 predictions[0] = np.around(predictions[0].astype(np.double))
cases_csv = np.concatenate(( cases_csv, predictions[:, :, 0]), axis=1)
 death = np.around(death.astype(np.double))
death[death < 0] = 0
death_csv = np.expand_dims(death[:, -1], axis=1)
 predictions[1] = np.around(predictions[1].astype(np.double))
death_csv = np.concatenate(( death_csv, predictions[:, :, 1]), axis=1)
 cases_csv = np.cumsum(cases_csv, axis=1)
 death_csv = np.cumsum(death_csv, axis=1)
 death_csv = death_csv[:, 1:]
cases_csv = cases_csv[:, 1:]
death_csv = np.concatenate(( death[:, -11:], death_csv), axis=1)
 cases_csv = np.concatenate(( cases[:, -11:], cases_csv), axis=1)
 csv = []
cases_csv = np.reshape(cases_csv[:, 1:],(-1, 1))
death_csv = np.reshape(death_csv[:, 1:],(-1, 1))
j = 1
for idx,(c, d)in enumerate(zip(cases_csv, death_csv)) :
csv.append([j, c, d])
 j += 1
print('done')
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
backtest2(sequences_train, model, given, total_missing )
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
sub = pd.read_csv('.. /input/submission2/submission.csv', header=None,dtype=np.float32)
 sub = pd.DataFrame(sub.values, columns =['ForecastId','ConfirmedCases','Fatalities'])
 sub.ConfirmedCases.astype(np.double)
 sub.Fatalities.astype(np.double)
 sub.ForecastId = sub.ForecastId.astype(np.int)
 sub.to_csv("submission.csv", index=False)
 print('done')