Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_pretrained>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model2 = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)
 try:
model2.load_state_dict(torch.load(".. /input/saint-plus-model/saint_plus_model_20210103.pt_v2"))
except:
model2.load_state_dict(torch.load(".. /input/saint-plus-model/saint_plus_model_20210103.pt_v2", map_location='cpu'))
model2.to(device)
 model2.eval()<load_pretrained>
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.utils import np_utils
from keras.optimizers import RMSprop
from keras.callbacks import ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
lt_correct_dict = pickle.load(open('.. /input/arvis-feature/last_timestamp_correct.pkl', 'rb'))
np_uq_td = pickle.load(open(".. /input/uq-data/np_uq_td_0518.pkl.data","rb"))
curr_u_dict = pickle.load(open(".. /input/uq-data/curr_u_dict_0614_only_user_three_time_diff.pkl.data","rb"))
max_timestamp_u_dict = pickle.load(open(".. /input/arvis-feature/max_timestamp_u_dict_2015.pkl","rb"))
max_timestamp_u_dict2 = pickle.load(open(".. /input/arvis-feature/max_timestamp_u_dict2_2015.pkl","rb"))
max_timestamp_u_dict3 = pickle.load(open(".. /input/arvis-feature/max_timestamp_u_dict3_2015.pkl","rb"))<feature_engineering>
df_train_x = df_train.iloc[:,1:]
df_train_y = df_train.iloc[:,:1]
def add_uq_feats_and_update(df):
conn = sqlite3.connect('user_ques_db.db')
 cursor = conn.cursor()
global idx
uq_timediff = np.zeros(len(df), dtype=np.uint64)
 for cnt,row in enumerate(df[['user_id','content_id','timestamp']].itertuples(index=False)) :
cursor.execute(f'select idx from user where user_id = {row[0]} and content_id = {row[1]}')
 tmp_idx = cursor.fetchall()
if tmp_idx == []:
uq_timediff[cnt] = 0
np_uq_td[idx] = row[2]
cursor.execute(f'insert into user(user_id, content_id, idx)values({row[0]}, {row[1]}, {idx})')
 idx += 1
else:
tmp_idx = tmp_idx[0][0]
uq_timediff[cnt] = row[2] - np_uq_td[tmp_idx]
np_uq_td[tmp_idx] = row[2]
cursor.close()
conn.commit()
conn.close()
uq_feats_df = pd.DataFrame({'curr_uq_time_diff':uq_timediff})
 df = pd.concat([df, uq_feats_df], axis=1)
 return df<feature_engineering>
def cnn_model(result_class_size):
model = Sequential()
model.add(Conv2D(32,(5, 5), input_shape=(28,28,1), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(16,(3, 3), activation='relu'))
model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(130, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(result_class_size, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=RMSprop() , metrics=['accuracy'])
 return model
def add_user_feats_without_update(df):
utdiff = np.zeros(len(df), dtype=np.uint64)
 utdiff_mean = np.zeros(len(df), dtype=np.uint64)
 uelapdiff = np.zeros(len(df), dtype=np.float32)
 for cnt,row in enumerate(df[['user_id','timestamp','prior_question_elapsed_time']].itertuples(index=False)) :
if row[0] in curr_u_dict:
utdiff[cnt] = row[1] - curr_u_dict[row[0]]["uts"]
utdiff_mean[cnt] = curr_u_dict[row[0]]["utsdiff"][1] / curr_u_dict[row[0]]["utsdiff"][0]
uelapdiff[cnt] = row[2] - curr_u_dict[row[0]]["uelapdiff"]
else:
utdiff[cnt] = 0; utdiff_mean[cnt] = 0; uelapdiff[cnt] = 0;
user_feats_df = pd.DataFrame({'curr_user_time_diff':utdiff, 'curr_user_time_diff_mean':utdiff_mean,
'curr_user_elapsed_time_diff':uelapdiff
})
 user_feats_df['curr_user_elapsed_time_diff'].fillna(0, inplace=True)
 df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df):
for cnt,row in enumerate(df[['user_id','content_id','answered_correctly','timestamp','prior_question_elapsed_time']].itertuples(index=False)) :
if row[0] in curr_u_dict:
curr_u_dict[row[0]]["uts"] = row[3]
curr_u_dict[row[0]]["utsdiff"][0] += 1
curr_u_dict[row[0]]["utsdiff"][1] += row[3]
curr_u_dict[row[0]]["uelapdiff"] = row[4]
else:
curr_u_dict[row[0]] = {}
curr_u_dict[row[0]]["uts"] = row[3]
curr_u_dict[row[0]]["utsdiff"] = [1, row[3]]
curr_u_dict[row[0]]["uelapdiff"] = row[4]<categorify>
df_test = df_test / 255
df_train_x = df_train_x / 255
def add_user_feats(df):
utdiff = np.zeros(len(df), dtype=np.uint64)
 utdiff_mean = np.zeros(len(df), dtype=np.uint64)
 uelapdiff = np.zeros(len(df), dtype=np.float32)
 for cnt,row in enumerate(tqdm(df[['user_id','content_id','answered_correctly',
'timestamp','prior_question_elapsed_time',
]].itertuples(index=False),total=df.shape[0])) :
if row[0] in curr_u_dict:
utdiff[cnt] = row[3] - curr_u_dict[row[0]]["uts"]
utdiff_mean[cnt] = curr_u_dict[row[0]]["utsdiff"][1] / curr_u_dict[row[0]]["utsdiff"][0]
uelapdiff[cnt] = row[4] - curr_u_dict[row[0]]["uelapdiff"]
curr_u_dict[row[0]]["uts"] = row[3]
curr_u_dict[row[0]]["utsdiff"][0] += 1
curr_u_dict[row[0]]["utsdiff"][1] += row[3]
curr_u_dict[row[0]]["uelapdiff"] = row[4]
else:
utdiff[cnt] = 0; utdiff_mean[cnt] = 0; uelapdiff[cnt] = 0;
curr_u_dict[row[0]] = {}
curr_u_dict[row[0]]["uts"] = row[3]
curr_u_dict[row[0]]["utsdiff"] = [1, row[3]]
curr_u_dict[row[0]]["uelapdiff"] = row[4]
user_feats_df = pd.DataFrame({
'curr_user_time_diff':utdiff, 'curr_user_time_diff_mean':utdiff_mean,
'curr_user_elapsed_time_diff':uelapdiff
})
 user_feats_df['curr_user_elapsed_time_diff'].fillna(0, inplace=True)
 df = pd.concat([df, user_feats_df], axis=1)
 return df 
<categorify>
arr_train_x_28x28 = np.reshape(df_train_x.values,(df_train_x.values.shape[0], 28, 28, 1))
arr_test_x_28x28 = np.reshape(df_test.values,(df_test.values.shape[0], 28, 28, 1))
def lagtime_for_test(df):
lagtime_mean = 0
lagtime_mean2 = 0
lagtime_mean3 = 0
lagtime = np.zeros(len(df), dtype=np.float32)
 lagtime2 = np.zeros(len(df), dtype=np.float32)
 lagtime3 = np.zeros(len(df), dtype=np.float32)
 for i,(user_id,
content_type_id,
timestamp,
content_id,)in enumerate(zip(df['user_id'].values, df['content_type_id'].values, df['timestamp'].values, df['content_id'].values)) :
if content_type_id==0:
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 df["lag_time"]= lagtime
df["lag_time2"]= lagtime2
df["lag_time3"]= lagtime3
df["lag_time"].fillna(-1, inplace=True)
 df["lag_time2"].fillna(-1, inplace=True)
 df["lag_time3"].fillna(-1, inplace=True)
 df['lag_time'] = df['lag_time'].replace(0, method="ffill")
 df['lag_time2'] = df['lag_time2'].replace(0, method="ffill")
 df['lag_time3'] = df['lag_time3'].replace(0, method="ffill")
 df["lag_time"] = df["lag_time"].astype("uint64")
 df["lag_time2"] = df["lag_time2"].astype("uint64")
 df["lag_time3"] = df["lag_time3"].astype("uint64")
 return df<categorify>
random_seed = 3
split_train_x, split_val_x, split_train_y, split_val_y, = train_test_split(arr_train_x_28x28, arr_train_y, test_size = 0.08, random_state=random_seed )
def add_feats(df_np, feat_dict, col_idx, col_feat):

 current_feat_value = np.zeros(len(df_np))
for cnt, row in enumerate(df_np[:,[col_idx, col_feat]]):
current_feat_value[cnt] = feat_dict[row[0]]
feat_dict[row[0]] += row[1]
df_np[:, col_feat] = current_feat_value
return df_np
def add_feats_from_dict(df_np, feat_dict, col_idx, col_dict=-1):

 current_feat_value = np.zeros(len(df_np))
for cnt, idx in enumerate(df_np[:,col_idx]):
if col_dict == -1: current_feat_value[cnt] = feat_dict[idx]
else: current_feat_value[cnt] = feat_dict[idx][col_dict]
return(np.c_[ df_np, current_feat_value ])
 def add_feats_from_dict_got_new_user(df_np, feat_dict, col_idx, col_dict=-1):

 current_feat_value = np.zeros(len(df_np))
for cnt, idx in enumerate(df_np[:,col_idx]):
if idx in feat_dict.keys() :
if col_dict == -1: current_feat_value[cnt] = feat_dict[idx]
else: current_feat_value[cnt] = feat_dict[idx][col_dict]
else:
if col_dict == -1: current_feat_value[cnt] = feat_dict[-100]
else: current_feat_value[cnt] = feat_dict[-100][col_dict]
feat_dict[idx] = feat_dict[-100]
return(np.c_[ df_np, current_feat_value ])
 def update_dict(df_pd, feat_dict, col_idx, col_feat, col_dict=-1):
for row in df_pd[['content_type_id', col_idx, col_feat]].values:
if row[0] == 0:
if col_dict == -1: feat_dict[row[1]] += row[2]
else: feat_dict[row[1]][col_dict] += row[2]
def add_ohe(df, col_feat, oh_value):
return(np.c_[ df, np.array([int(i == oh_value)for i in df[:,col_feat]])])
 def user_slice_accuracy_n_get(df_pd, feat_dict):
global_first_question_accuracy = 0.6453965159034877
current_list = np.zeros(len(df_pd))
for cnt,(user, content_type_id)in enumerate(df_pd[['user_id', 'content_type_id']].values):
if content_type_id == 0:
if user in feat_dict:
current_list[cnt] = np.mean(feat_dict[user])
 else:
current_list[cnt] = 0.6454
else:
current_list[cnt] = 0
return current_list
def user_slice_accuracy_n_update(df_pd, feat_dict, border=5):
for cnt,(user, answer)in enumerate(df_pd[['user_id', 'answered_correctly']].values):
if user in feat_dict:
feat_dict[user].append(answer)
 feat_dict[user] = feat_dict[user][-border:]
else:
feat_dict[user] = [answer]
return feat_dict
def user_slice_accuracy_session_get(df_pd, feat_dict, session_max_time=12):
current_list = np.zeros(len(df_pd))
for cnt,(user, timestamp, content_type_id)in enumerate(df_pd[['user_id', 'timestamp', 'content_type_id']].values):
if content_type_id == 0:
if user in feat_dict:
time_delta_h =(timestamp - feat_dict[user][1])/ 1000 / 60 / 60
if time_delta_h < session_max_time:
current_list[cnt] = np.mean(feat_dict[user][0])
 else:
current_list[cnt] = 0.67
else:
current_list[cnt] = 0.67
else: current_list[cnt] = 0
return current_list
def user_slice_accuracy_session_update(df_pd, feat_dict, session_max_time=12):
for cnt,(user, answer, timestamp)in enumerate(df_pd[['user_id', 'answered_correctly', 'timestamp']].values):
if user in feat_dict:
time_delta_h =(timestamp - feat_dict[user][1])/ 1000 / 60 / 60
if time_delta_h < session_max_time:
feat_dict[user][0].append(answer)
 feat_dict[user][1] = timestamp
else:
feat_dict[user][0] = [answer]
feat_dict[user][1] = timestamp
else:
feat_dict[user] = [[answer], timestamp]
return feat_dict
def user_question_attempt_cnt_get_update(df_pd, feat_dict):
current_feat_value = np.zeros(len(df_pd))
for idx,(user_id, content_id, content_type_id)in enumerate(df_pd[['user_id', 'content_id', 'content_type_id']].values):
if content_type_id == 0:
current_feat_value[idx] = list(feat_dict[user_id] ).count(content_id)
 feat_dict[user_id] = np.append(feat_dict[user_id], content_id)
 else:
current_feat_value[idx] = 0
return current_feat_value
def user_lectures_part(df_pd, feat_dict):
current_list = np.zeros(len(df_pd))
for cnt,(user, content_type_id, part_q, part_l)in enumerate(df_pd[['user_id', 'content_type_id', 'part_q', 'part_l']].values):
part_q = max(0, part_q)
 if content_type_id == 0:
if user in feat_dict:
current_list[cnt] = feat_dict[user][part_q]
else:
feat_dict[user] = [0] * 8
current_list[cnt] = 0
else:
if user in feat_dict:
feat_dict[user][part_l] += 1
else:
feat_dict[user] = [0] * 8
feat_dict[user][part_l] += 1
current_list[cnt] = 0
return current_list
def user_lecture_cnt(df_pd, feat_dict):

 current_list = np.zeros(len(df_pd))
for cnt, user in enumerate(df_pd['user_id'].values):
if user in feat_dict:
current_list[cnt] = sum(feat_dict[user])
 else:
current_list[cnt] = 0
return current_list
def user_l_q_tag_equal(df_pd, feat_dict):

 current_list = np.zeros(len(df_pd))
for idx,(user, content_type_id, tag, tags_list)in enumerate(df_pd[['user_id', 'content_type_id', 'tag_l', 'tags_list']].values):
if content_type_id == 0:
current_list[idx] = len(set(feat_dict[user])& set(tags_list))
else:
feat_dict[user].append(int(tag))
current_list[idx] = 0
return current_list
def get_q_l(df_pd, feat_dict, key):
current_list = []
for idx, content_id in enumerate(df_pd['content_id'].values):
if content_id in feat_dict:
current_list.append(feat_dict[content_id][key])
 else:
current_list.append(-100)
 return current_list
def dict_user_timestampsdelta_get_update_3(df, feat_dict):
q_list = np.zeros(( len(df), 3), dtype = np.float32)
 l_list = np.zeros(( len(df), 3), dtype = np.float32)
 for cnt,(user_id, timestamp, content_type_id)in enumerate(df[['user_id', 'timestamp', 'content_type_id']].values):
timestamp = timestamp / 1000 / 60
if user_id in feat_dict:
q_list[cnt] = np.array([timestamp - t for t in feat_dict[user_id][0]])
 l_list[cnt] = np.array([timestamp - t for t in feat_dict[user_id][1]])
 feat_dict[user_id][int(content_type_id)].pop(0)
 feat_dict[user_id][int(content_type_id)].append(timestamp)
 else:
if content_type_id == 1: feat_dict[user_id] = [[np.nan, np.nan, np.nan], [np.nan, np.nan, 0]]
else: feat_dict[user_id] = [[np.nan, np.nan, 0], [np.nan, np.nan, np.nan]]
q_list[cnt] = np.array([np.nan, np.nan, np.nan])
 l_list[cnt] = np.array([np.nan, np.nan, np.nan])
 for i in [0, 1, 2]:
df['prior_question_' + str(i+1)+ '_timedelta_min'] = q_list[:, i]
df['prior_lecture_' + str(i+1)+ '_timedelta_min'] = l_list[:, i]
df['prior_question_' + str(i+1)+ '_timedelta_min'] = df['prior_question_' + str(i+1)+ '_timedelta_min'].fillna(-100 ).replace(0, method='ffill')
 df['prior_lecture_' + str(i+1)+ '_timedelta_min'] = df['prior_lecture_' + str(i+1)+ '_timedelta_min'].fillna(-100 ).replace(0, method='ffill')
 del [q_list, l_list]
return df
def user_question_tag_accuracy_get(df, feat_calc):
tags_w = [0.43, 0.27, 0.18, 0.08, 0.03, 0.01]
values = np.zeros(len(df))
feat_list = ['user_id', 'content_type_id', 'tags_list']
for cnt,(user, content_type_id, tags_list)in enumerate(df[feat_list].values):
if tags_list == -100: tags_list = [-100]
if content_type_id == 0:
if user in feat_calc:
user_tags_accuracy = 0
for tag_i in tags_list:
tags_accuracy_list = []
if tag_i in feat_calc[user][0]:
tag_i_idx = feat_calc[user][0].index(tag_i)
 tags_accuracy_list.append(feat_calc[user][2][tag_i_idx] / feat_calc[user][1][tag_i_idx])
 else:
tags_accuracy_list.append(dict_global_question_tag_accuracy[tag_i][2])
 l = len(tags_accuracy_list)
 tags_w_l = tags_w[:l]
tags_w_l_sum = sum(tags_w_l)
 tags_w_current = [x/tags_w_l_sum for x in tags_w_l]
user_tags_accuracy = sum([tag * w for tag, w in zip(tags_accuracy_list, tags_w_current)])
 else:
for tag_i in tags_list:
tags_accuracy_list = []
tags_accuracy_list.append(dict_global_question_tag_accuracy[tag_i][2])
 l = len(tags_accuracy_list)
 tags_w_l = tags_w[:l]
tags_w_l_sum = sum(tags_w_l)
 tags_w_current = [x/tags_w_l_sum for x in tags_w_l]
user_tags_accuracy = sum([tag * w for tag, w in zip(tags_accuracy_list, tags_w_current)])
 values[cnt] = user_tags_accuracy
else:
values[cnt] = 0
return values
def user_question_tag_accuracy_update(df, feat_calc):
feat_list = ['user_id', 'answered_correctly', 'content_type_id', 'tags_list']
for cnt,(user, answer, content_type_id, tags_list)in enumerate(df[feat_list].values):
if tags_list == -100: tags_list = [-100]
if content_type_id == 0:
if user in feat_calc:
user_tags_accuracy = 0
for tag_i in tags_list:
tags_accuracy_list = []
if tag_i in feat_calc[user][0]:
tag_i_idx = feat_calc[user][0].index(tag_i)
 feat_calc[user][1][tag_i_idx] += 1
feat_calc[user][2][tag_i_idx] += answer
else:
feat_calc[user][0].append(tag_i)
 feat_calc[user][1].append(1)
 feat_calc[user][2].append(answer)
 else:
feat_calc[user] = [[], [], []]
for tag_i in tags_list:
feat_calc[user][0].append(tag_i)
 feat_calc[user][1].append(1)
 feat_calc[user][2].append(answer)
 return feat_calc
def user_correct_incorrect_timestamp_get(df, feat_dict):

 incorrect_list = []
correct_list = []
for(user_id, timestamp, content_type_id)in df[['user_id', 'timestamp', 'content_type_id']].values:
timestamp = timestamp / 1000 / 60
correct_value, incorrect_value = 0, 0
if content_type_id:
incorrect_list.append(-100)
 correct_list.append(-100)
 else:
if user_id in feat_dict:
incorrect_value, correct_value = feat_dict[user_id][0], feat_dict[user_id][1]
else:
incorrect_value, correct_value = np.nan, np.nan
incorrect_value =(timestamp - incorrect_value)
 correct_value =(timestamp - correct_value)
 incorrect_list.append(incorrect_value)
 correct_list.append(correct_value)
 df['prior_question_incorrect_timedelta_min'] = incorrect_list
df['prior_question_correct_timedelta_min'] = correct_list
df['prior_question_incorrect_timedelta_min'] = df['prior_question_incorrect_timedelta_min'].fillna(-100 ).replace(0, method='ffill')
 df['prior_question_correct_timedelta_min'] = df['prior_question_correct_timedelta_min'].fillna(-100 ).replace(0, method='ffill')
 return df
def user_correct_incorrect_timestamp_update(df, feat_dict):

 incorrect_list = []
correct_list = []
for(user_id, timestamp, content_type_id, answer)in df[['user_id', 'timestamp', 'content_type_id', 'answered_correctly']].values:
timestamp = timestamp / 1000 / 60
if content_type_id == 0:
if user_id in feat_dict:
if answer: feat_dict[user_id][1] = timestamp
else: feat_dict[user_id][0] = timestamp
else:
if answer: feat_dict[user_id] = [np.nan, timestamp]
else: feat_dict[user_id] = [timestamp, np.nan]
return feat_dict
time_session_map = {'q_count_all' : 0,
'q_count_n' : 1,
'time_all_n' : 2,
'time_n' : 3,
'time_dict_all' : 4,
'time_dict_n' : 5,
'prior_timestamp' : 6,
'prior_container' : 7,
'prior_container_shape' : 8,
'prior_content_id' : 9,
}
def user_slice_question_time_mean_session(df_pd, feat_dict, session_max_time_min = 180):
prior_question_elapsed_time_mean = 25452.541
out_mean_n = np.zeros(len(df_pd))
out_mean_all = np.zeros(len(df_pd))
out_delta_n = np.zeros(len(df_pd))
out_delta_all = np.zeros(len(df_pd))
calc_list = ['user_id', 'timestamp', 'task_container_id', 'content_id',
'content_type_id', 'prior_question_elapsed_time', 'task_container_freq']
for cnt,(user, timestamp, task_container_id, content_id, content_type_id, prior_question_elapsed_time, task_container_freq)in enumerate(df_pd[calc_list].values):
timestamp = timestamp / 1000 / 60
if content_id not in dict_content_elapsed_time_mean: content_id = -100
if content_type_id == 0:
if user in feat_dict:
time_delta =(timestamp - feat_dict[user][time_session_map['prior_timestamp']])
 if time_delta < session_max_time_min:
if task_container_id != feat_dict[user][time_session_map['prior_container']]:
s = feat_dict[user][time_session_map['prior_container_shape']]
c = feat_dict[user][time_session_map['prior_content_id']]
feat_dict[user][time_session_map['time_all_n']] += prior_question_elapsed_time * s
feat_dict[user][time_session_map['time_n']] += prior_question_elapsed_time * s
feat_dict[user][time_session_map['q_count_all']] += s
feat_dict[user][time_session_map['q_count_n']] += s
feat_dict[user][time_session_map['time_dict_all']] += dict_content_elapsed_time_mean[c] * s
feat_dict[user][time_session_map['time_dict_n']] += dict_content_elapsed_time_mean[c] * s
feat_dict[user][time_session_map['prior_timestamp']] = timestamp
feat_dict[user][time_session_map['prior_container']] = task_container_id
feat_dict[user][time_session_map['prior_content_id']] = content_id
feat_dict[user][time_session_map['prior_container_shape']] = task_container_freq
out_mean_n[cnt] = feat_dict[user][time_session_map['time_n']] / feat_dict[user][time_session_map['q_count_n']]
out_mean_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['q_count_all']]
out_delta_n[cnt] = feat_dict[user][time_session_map['time_n']] / feat_dict[user][time_session_map['time_dict_n']]
out_delta_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['time_dict_all']]
else:
out_mean_n[cnt] = out_mean_n[cnt-1]
out_mean_all[cnt] = out_mean_all[cnt-1]
out_delta_n[cnt] = out_delta_n[cnt-1]
out_delta_all[cnt] = out_delta_all[cnt-1]
else:
if task_container_id != feat_dict[user][time_session_map['prior_container']]:
s = feat_dict[user][time_session_map['prior_container_shape']]
c = feat_dict[user][time_session_map['prior_content_id']]
feat_dict[user][time_session_map['time_all_n']] += prior_question_elapsed_time * s
feat_dict[user][time_session_map['time_n']] = 0
feat_dict[user][time_session_map['q_count_all']] += s
feat_dict[user][time_session_map['q_count_n']] = 0
feat_dict[user][time_session_map['time_dict_all']] += dict_content_elapsed_time_mean[c] * s
feat_dict[user][time_session_map['time_dict_n']] = 0
feat_dict[user][time_session_map['prior_timestamp']] = timestamp
feat_dict[user][time_session_map['prior_container']] = task_container_id
feat_dict[user][time_session_map['prior_content_id']] = content_id
feat_dict[user][time_session_map['prior_container_shape']] = task_container_freq
out_mean_n[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['q_count_all']]
out_mean_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['q_count_all']]
out_delta_n[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['time_dict_all']]
out_delta_all[cnt] = feat_dict[user][time_session_map['time_all_n']] / feat_dict[user][time_session_map['time_dict_all']]
else:
out_mean_n[cnt] = out_mean_n[cnt-1]
out_mean_all[cnt] = out_mean_all[cnt-1]
out_delta_n[cnt] = out_delta_n[cnt-1]
out_delta_all[cnt] = out_delta_all[cnt-1]
else:
out_mean_n[cnt] = prior_question_elapsed_time_mean
out_mean_all[cnt] = prior_question_elapsed_time_mean
out_delta_n[cnt] = prior_question_elapsed_time_mean / dict_content_elapsed_time_mean[content_id]
out_delta_all[cnt] = prior_question_elapsed_time_mean / dict_content_elapsed_time_mean[content_id]
feat_dict[user] = [1, 1, 0, 0,
dict_content_elapsed_time_mean[content_id], dict_content_elapsed_time_mean[content_id],
timestamp, task_container_id, task_container_freq, content_id]
else: out_mean_n[cnt], out_mean_all[cnt], out_delta_n[cnt], out_delta_all[cnt] = 0, 0, 0, 0
df_pd['user_question_time_mean_n_session'] = out_mean_n
df_pd['user_question_time_mean_all_session'] = out_mean_all
df_pd['user_question_time_delta_n_session'] = out_delta_n
df_pd['user_question_time_delta_all_session'] = out_delta_all
return df_pd
user_priorq_expl_types_map = {'q_count' : 0,
'c_t_cnt' : 1,
'c_f_cnt' : 2,
'w_t_cnt' : 3,
'w_f_cnt' : 4,
'prior_container' : 5,
'prior_container_shape' : 6,
'prior_answer_expl_type' : 7,
}
def user_priorq_expl_types_get(df_pd, feat_dict):
c_t_cnt = np.zeros(len(df_pd))
c_f_cnt = np.zeros(len(df_pd))
w_t_cnt = np.zeros(len(df_pd))
w_f_cnt = np.zeros(len(df_pd))
cw_tf_type = np.zeros(len(df_pd))
calc_list = ['user_id', 'task_container_id', 'content_type_id',
'prior_question_had_explanation', 'task_container_freq']
for cnt,(user, task_container_id, content_type_id, priorq_had_expl, task_container_freq)in enumerate(df_pd[calc_list].values):
if content_type_id == 0:
if user in feat_dict:
if task_container_id != feat_dict[user][user_priorq_expl_types_map['prior_container']]:
prior_c_shape = feat_dict[user][user_priorq_expl_types_map['prior_container_shape']]
q_count = feat_dict[user][user_priorq_expl_types_map['q_count']] + 0.0001
c_t_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['c_t_cnt']] / q_count
c_f_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['c_f_cnt']] / q_count
w_t_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['w_t_cnt']] / q_count
w_f_cnt[cnt] = feat_dict[user][user_priorq_expl_types_map['w_f_cnt']] / q_count
cw_tf_type[cnt] = feat_dict[user][user_priorq_expl_types_map['prior_answer_expl_type']]
else:
c_t_cnt[cnt] = c_t_cnt[cnt - 1]
c_f_cnt[cnt] = c_f_cnt[cnt - 1]
w_t_cnt[cnt] = w_t_cnt[cnt - 1]
w_f_cnt[cnt] = w_f_cnt[cnt - 1]
cw_tf_type[cnt] = cw_tf_type[cnt - 1]
else:
c_t_cnt[cnt], c_f_cnt[cnt], w_t_cnt[cnt], w_f_cnt[cnt], w_f_cnt[cnt] = 0, 0, 0, 0, 0
else: c_t_cnt[cnt], c_f_cnt[cnt], w_t_cnt[cnt], w_f_cnt[cnt], cw_tf_type[cnt] = 0, 0, 0, 0, 0
df_pd['user_prior_correct_expl_prc'] = c_t_cnt
df_pd['user_prior_correct_noexpl_prc'] = c_f_cnt
df_pd['user_prior_wrong_expl_prc'] = w_t_cnt
df_pd['user_prior_wrong_noexpl_prc'] = w_f_cnt
df_pd['user_prior_answer_expl_type'] = cw_tf_type
return df_pd
def user_priorq_expl_types_update(df_pd, feat_dict):
calc_list = ['user_id', 'task_container_id', 'content_type_id',
'prior_question_had_explanation', 'answered_correctly', 'task_container_freq']
for cnt,(user, task_container_id, content_type_id, priorq_had_expl, answer, task_container_freq)in enumerate(df_pd[calc_list].values):
if answer: t1 = 'c_'
else: t1 = 'w_'
if priorq_had_expl: t2 = 't_cnt'
else: t2 = 'f_cnt'
col = t1 + t2
if content_type_id == 0:
if user in feat_dict:
if task_container_id != feat_dict[user][user_priorq_expl_types_map['prior_container']]:
prior_c_shape = feat_dict[user][user_priorq_expl_types_map['prior_container_shape']]
feat_dict[user][user_priorq_expl_types_map['q_count']] += prior_c_shape
feat_dict[user][user_priorq_expl_types_map[col]] += prior_c_shape
feat_dict[user][user_priorq_expl_types_map['prior_container']] = task_container_id
feat_dict[user][user_priorq_expl_types_map['prior_container_shape']] = task_container_freq
feat_dict[user][user_priorq_expl_types_map['prior_answer_expl_type']] = user_priorq_expl_types_map[col]
else:
feat_dict[user] = [0, 0, 0, 0, 0,
task_container_id, task_container_freq, 0]
return feat_dict
def user_lectures_typeof_cnt(df, feat_dict):
concept = np.zeros(len(df))
solving_question = np.zeros(len(df))
for cnt,(user, content_id, content_type_id)in enumerate(df[['user_id', 'content_id', 'content_type_id']].values):
if content_type_id:
if user in feat_dict:
if content_id in dict_lectures:
if dict_lectures[content_id]['type_of'] == 'concept': feat_dict[user][0] += 1
elif dict_lectures[content_id]['type_of'] == 'solving question': feat_dict[user][1] += 1
else:
if content_id in dict_lectures:
feat_dict[user] = [0, 0]
if dict_lectures[content_id]['type_of'] == 'concept': feat_dict[user][0] += 1
elif dict_lectures[content_id]['type_of'] == 'solving question': feat_dict[user][1] += 1
concept[cnt] = 0
solving_question[cnt] = 0
else:
if user in feat_dict:
concept[cnt] = feat_dict[user][0]
solving_question[cnt] = feat_dict[user][1]
else:
concept[cnt] = 0
solving_question[cnt] = 0
df['lecture_concept_cnt'] = concept.astype(np.uint16)
 df['lecture_solving_question_cnt'] = solving_question.astype(np.uint16)
 return df
def user_answer_mode_n_get(df_pd, feat_dict, border=10):
current_list = np.zeros(len(df_pd), dtype = np.uint8)
 for cnt,(user, content_type_id, content_id)in enumerate(df_pd[['user_id', 'content_type_id', 'content_id']].values):
if content_type_id == 0:
if user in feat_dict:
if content_id in dict_questions:
current_list[cnt] = max(set(feat_dict[user]), key=feat_dict[user].count)== dict_questions[content_id]['correct_answer']
else:
current_list[cnt] = 100
else:
current_list[cnt] = 100
else:
current_list[cnt] = 0
return current_list
def user_answer_mode_n_update(df_pd, feat_dict, border=10):
for cnt,(user, user_answer, content_type_id, content_id)in enumerate(df_pd[['user_id', 'user_answer', 'content_type_id', 'content_id']].values):
if content_type_id == 0:
feat_dict[user].append(user_answer)
 feat_dict[user] = feat_dict[user][-border:]
return feat_dict
first_bundle_id_map = {7900 : 1,
128 : 2,
5692 : 3,
-100 : 4,
}
def question_bundle_id_get(df_pd, feat_dict):
current_list = np.zeros(len(df_pd))
for cnt,(content_type_id, content_id)in enumerate(df_pd[['content_type_id', 'content_id']].values):
if content_type_id == 0:
if content_id in feat_dict:
current_list[cnt] = feat_dict[content_id]['bundle_id']
else:
current_list[cnt] = -100
else:
current_list[cnt] = 0
df_pd['bundle_id'] = current_list.astype(np.int32)
 return df_pd
def user_bundle_cluster_get_update(df_pd, feat_dict):
l = ['user_id', 'content_type_id', 'bundle_id', 'timestamp']
current_list = np.zeros(len(df_pd))
for cnt,(user, content_type_id, bundle, timestamp)in enumerate(df_pd[l].values):
if user not in feat_dict:
if content_type_id == 0 and timestamp == 0:
if bundle in first_bundle_id_map:
feat_dict[user] = first_bundle_id_map[bundle]
else: feat_dict[user] = 4
else: feat_dict[user] = 4
current_list[cnt] = feat_dict[user]
df_pd['first_bundle_id_cluster'] = current_list.astype(np.uint8)
 return df_pd
def question_bundle_accuracy_update(df_pd, feat_dict):
l = ['content_type_id', 'bundle_id', 'answered_correctly', 'first_bundle_id_cluster']
for cnt,(content_type_id, bundle_id, answer, cluster)in enumerate(df_pd[l].values):
if content_type_id == 0:
if bundle_id in feat_dict:
feat_dict[bundle_id][0][0] += 1
feat_dict[bundle_id][0][1] += answer
feat_dict[bundle_id][cluster][0] += 1
feat_dict[bundle_id][cluster][1] += answer
else:
feat_dict[bundle_id] = [[1, answer], [0, 0], [0, 0], [0, 0], [1, answer]]
return feat_dict
def question_bundle_accuracy_get(df_pd, feat_dict):
current_list = np.zeros(( len(df_pd), 2), dtype = np.float32)
 l = ['content_type_id', 'bundle_id', 'first_bundle_id_cluster']
for cnt,(content_type_id, bundle_id, cluster)in enumerate(df_pd[l].values):
if content_type_id == 0:
if bundle_id in feat_dict:
current_list[cnt, 0] = feat_dict[bundle_id][0][1] /(feat_dict[bundle_id][0][0] + 0.000001)
 current_list[cnt, 1] = feat_dict[bundle_id][cluster][1] /(feat_dict[bundle_id][cluster][0] + 0.000001)
 else:
current_list[cnt, 0] = 0.67
current_list[cnt, 1] = 0.67
else:
current_list[cnt, 0] = 0
current_list[cnt, 1] = 0
df_pd['bundle_id_all_accuracy'] = current_list[:,0].astype(np.float32)
 df_pd['bundle_id_cluster_accuracy'] = current_list[:,1].astype(np.float32)
 return df_pd<define_variables>
reduce_lr = ReduceLROnPlateau(monitor='val_acc',
factor=0.5,
patience=3,
min_lr=0.00001 )
question_part_map = {
'part_-100_count' : 0,
'part_-100_count_correct' : 1,
'part_-100_accuracy' : 2,
'part_1_count' : 3,
'part_1_count_correct' : 4,
'part_1_accuracy' : 5,
'part_2_count' : 6,
'part_2_count_correct' : 7,
'part_2_accuracy' : 8,
'part_3_count' : 9,
'part_3_count_correct' : 10,
'part_3_accuracy' : 11,
'part_4_count' : 12,
'part_4_count_correct' : 13,
'part_4_accuracy' : 14,
'part_5_count' : 15,
'part_5_count_correct' : 16,
'part_5_accuracy' : 17,
'part_6_count' : 18,
'part_6_count_correct' : 19,
'part_6_accuracy' : 20,
'part_7_count' : 21,
'part_7_count_correct' : 22,
'part_7_accuracy' : 23
}
question_part_def_accuracy = {
1 : 0.745,
2 : 0.709,
3 : 0.701,
4 : 0.631,
5 : 0.610,
6 : 0.669,
7 : 0.660,
-100 : 0.6,
}
def dict_user_question_part_accuracy_get(df_pd, feat_dict):
current_list = np.zeros(len(df_pd))
for idx,(user, part_q, content_type_id)in enumerate(df_pd[['user_id', 'part_q', 'content_type_id']].values):
part_q = int(part_q)
 if content_type_id == 0:
map_accuracy = question_part_map['part_' + str(part_q)+ '_accuracy']
if user in feat_dict:
current_list[idx] = feat_dict[user][map_accuracy]
else:
current_list[idx] = question_part_def_accuracy[part_q]
else:
current_list[idx] = 0
return current_list
def dict_user_question_part_accuracy_update(df_pd, feat_dict, trust_border=10):
for _,(user, part_q, content_type_id, ans_corr)in enumerate(df_pd[['user_id', 'part_q', 'content_type_id', 'answered_correctly']].values):
part_q = int(part_q)
 if content_type_id == 0:
map_cnt = question_part_map['part_' + str(part_q)+ '_count']
map_cnt_correct = question_part_map['part_' + str(part_q)+ '_count_correct']
map_accuracy = question_part_map['part_' + str(part_q)+ '_accuracy']
if user in feat_dict:
feat_dict[user][map_cnt] += 1
feat_dict[user][map_cnt_correct] += ans_corr
feat_dict[user][map_accuracy] = feat_dict[user][map_cnt_correct] / feat_dict[user][map_cnt]
if feat_dict[user][map_cnt] < trust_border:
feat_dict[user][map_accuracy] =(( feat_dict[user][map_accuracy] * feat_dict[user][map_cnt] +
question_part_def_accuracy[part_q] *(trust_border - feat_dict[user][map_cnt])) / trust_border)
 else:
feat_dict[user] = [0] * len(question_part_map)
 for i in range(1, 7):
feat_dict[user][question_part_map['part_' + str(i)+ '_accuracy']] = question_part_def_accuracy[i]
feat_dict[user][map_cnt] += 1
feat_dict[user][map_cnt_correct] += ans_corr
feat_dict[user][map_accuracy] = feat_dict[user][map_cnt_correct] / feat_dict[user][map_cnt]
feat_dict[user][map_accuracy] =(( feat_dict[user][map_accuracy] * feat_dict[user][map_cnt] +
question_part_def_accuracy[part_q] *(trust_border - feat_dict[user][map_cnt])) / trust_border)
 return feat_dict<load_pretrained>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 datagen.fit(split_train_x )
def load_obj(name):
with open('.. /input/riiid-numpy-df-3/' + name + '.pkl', 'rb')as f:
return pickle.load(f )<load_pretrained>
model.fit_generator(datagen.flow(split_train_x,split_train_y, batch_size=64),
epochs = 30, validation_data =(split_val_x,split_val_y),
verbose = 2, steps_per_epoch=700
, callbacks=[reduce_lr] )
cat_model = CatBoostClassifier()
cat_model.load_model('.. /input/riiid-lgb-v1/cat_arvis_v4.cbm')
 lgb_model = lgb.Booster(model_file='.. /input/riiid-lgb-v1/model_lgb_7946_v8_full_data_arvis.txt' )<load_pretrained>
prediction = model.predict_classes(arr_test_x_28x28, verbose=0)
 data_to_submit = pd.DataFrame({"ImageId": list(range(1,len(prediction)+1)) , "Label": prediction})
 data_to_submit.to_csv("result.csv", header=True, index = False )
dict_lectures = load_obj('dict_lectures')
 dict_questions = load_obj('dict_questions')
 dict_question_user_cnt = load_obj('dict_question_user_cnt')
 dict_correct_answers_user_cnt = load_obj('dict_correct_answers_user_cnt')
 dict_question_explonation_user_cnt = load_obj('dict_question_explonation_user_cnt')
 dict_questionid_part_tag12_avgtarget = load_obj('dict_questionid_part_tag12_avgtarget_5')
 dict_user_question_attempt_cnt = load_obj('dict_user_question_attempt_cnt')
 dict_user_lectures_part = load_obj('dict_user_lectures_part')
 dict_user_question_part_accuracy = load_obj('dict_user_question_part_accuracy')
 dict_user_l_q_tag_equal = load_obj('dict_user_l_q_tag_equal')
 dict_user_slice_accuracy_5 = load_obj('dict_user_slice_accuracy_5')
 dict_user_slice_accuracy_20 = load_obj('dict_user_slice_accuracy_20')
 dict_user_slice_accuracy_50 = load_obj('dict_user_slice_accuracy_50')
 dict_user_slice_accuracy_session_3 = load_obj('dict_user_slice_accuracy_session_3')
 dict_user_slice_accuracy_session_12 = load_obj('dict_user_slice_accuracy_session_12')
 dict_user_slice_accuracy_session_48 = load_obj('dict_user_slice_accuracy_session_48')
 dict_user_timestampsdelta_3 = load_obj('dict_user_timestampsdelta_3')
 dict_global_question_tag_accuracy = load_obj('dict_global_question_tag_accuracy')
 dict_global_question_tag_accuracy[-100] = 0.64
dict_user_question_tag_accuracy = load_obj('dict_user_question_tag_accuracy')
 dict_user_correct_incorrect_timestamp = load_obj('dict_user_correct_incorrect_timestamp')
 dict_content_elapsed_time_mean = load_obj('dict_content_elapsed_time_mean')
 dict_user_slice_question_time_mean_session = load_obj('dict_user_slice_question_time_mean_session')
 dict_user_priorq_expl_types = load_obj('dict_user_priorq_expl_types')
 dict_user_lectures_typeof_cnt = load_obj('dict_user_lectures_typeof_cnt')
 dict_user_answer_mode_10 = load_obj('dict_user_answer_mode_10')
 dict_user_answer_mode_50 = load_obj('dict_user_answer_mode_50')
 dict_question_bundle_accuracy = load_obj('dict_question_bundle_accuracy')
 dict_user_bundle_cluster = load_obj('dict_user_bundle_cluster' )<define_variables>
start_idx = randrange(df_test.shape[0]-10 )