Kaggle competition: Digit Recognizer
WINDOW = config.ROOT_PATH
with open(f"{WINDOW}/user_all_count.pkl", "rb")as f:
user_all_count = pickle.load(f)
 with open(f"{WINDOW}/user_correct_window_200.pkl", "rb")as f:
user_correct_window_200 = pickle.load(f)
 with open(f"{WINDOW}/prior_question_elapsed_time_window_dict.pkl", "rb")as f:
prior_question_elapsed_time_window_dict = pickle.load(f)
 with open(f"{WINDOW}/prior_question_had_explanation_count.pkl", "rb")as f:
prior_question_had_explanation_count = pickle.load(f)
 with open(f"{WINDOW}/prior_question_had_explanation_window_dict.pkl", "rb")as f:
prior_question_had_explanation_window_dict = pickle.load(f)
 with open(f"{WINDOW}/timediff_window_dict.pkl", "rb")as f:
timediff_window_dict = pickle.load(f )<define_variables>
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
col1 = [f"work_q_tag_{i}_v3" for i in range(188)]
col2 = [f"cumsum_q_tag_{i}_v3" for i in range(188)]
col3 = [f"work_l_tag_{i}_v2" for i in range(188)]
user_agg_feats_c = col1 + col2 + col3
rate_col = [f"correct_rate_q_tag_{i}" for i in range(188)]<feature_engineering>
X = train.drop('label',axis = 1)
 y = train.label
def get_content_feature(_content_id):
idx = np.where(content_agg_feats_v[:,0] == _content_id)[0][0]
v = content_agg_feats_v[idx, 1:]
return v.tolist()
def get_user_feature(_user_id):
idx = np.where(user_agg_feats_v[:,0] == _user_id)[0]
if len(idx)== 0:
return np.zeros(user_agg_feats_v.shape[1] - 1)
 else:
idx = idx[0]
v = user_agg_feats_v[idx, 1:]
return v.tolist()
def get_timediff(row):
_timestamp = row["timestamp"]
_user_id = row["user_id"]
try:
return _timestamp - last_timestamp_dic[_user_id]
except KeyError:
return 0
def get_lgbm_window_feat(_user_id):
try:
v = prior_question_elapsed_time_window_dict[_user_id]
v = np.array(v)[~np.isnan(v)]
prior_question_elapsed_time_std_w200 = v.std(ddof=config.DDOF)
 prior_question_elapsed_time_avg_w200 = v.mean()
prior_question_had_explanation_std_w200 = np.std(prior_question_had_explanation_window_dict[_user_id], ddof=config.DDOF)
 prior_question_had_explanation_avg_w200 = np.mean(prior_question_had_explanation_window_dict[_user_id])
 timediff_std_w200 = np.std(timediff_window_dict[_user_id], ddof=config.DDOF)
 timediff_avg_w200 = np.mean(timediff_window_dict[_user_id])
 _prior_question_had_explanation_count = prior_question_had_explanation_count[_user_id]
except KeyError:
prior_question_elapsed_time_std_w200 = 0
prior_question_elapsed_time_avg_w200 = 0
prior_question_had_explanation_std_w200 = 0
prior_question_had_explanation_avg_w200 = 0
timediff_std_w200 = 0
timediff_avg_w200 = 0
_prior_question_had_explanation_count = 0
return [
prior_question_elapsed_time_std_w200,
prior_question_elapsed_time_avg_w200,
prior_question_had_explanation_std_w200,
prior_question_had_explanation_avg_w200,
timediff_std_w200,
timediff_avg_w200,
_prior_question_had_explanation_count,
]<prepare_x_and_y>
y = to_categorical(y )
def update_infomation(row):
global user_agg_feats_v
_user_id = row["user_id"]
_timestamp = row["timestamp"]
_content_id = row["content_id"]
_answered_correctly = row["answered_correctly"]
_content_type_id = row["content_type_id"]
try:
_prior_question_had_explanation = int(row["prior_question_had_explanation"])
 except TypeError:
_prior_question_had_explanation = 0
try:
_prior_question_elapsed_time = float(row["prior_question_elapsed_time"])
 except TypeError:
_prior_question_elapsed_time = 0
try:
_timediff = _timestamp - last_timestamp_dic[_user_id]
except KeyError:
_timediff = 0
last_timestamp_dic[_user_id] = _timestamp
if _content_type_id == 0:
n_work = q_ohe_dic[_content_id]
n_correct = n_work * _answered_correctly
n_lecture = np.zeros(188)
 else:
n_work = np.zeros(188)
 n_correct = np.zeros(188)
 n_lecture = l_ohe_dic[_content_id]
tag_feats = np.hstack([n_work, n_correct, n_lecture])
 idx = np.where(user_agg_feats_v[:,0] == _user_id)[0]
if len(idx)== 0:
append_v = np.hstack([_user_id, tag_feats] ).astype(int)
 user_agg_feats_v = np.vstack([user_agg_feats_v, append_v])
 else:
idx = idx[0]
user_agg_feats_v[idx, 1:] += tag_feats.astype(int)
 try:
prior_question_had_explanation_count[_user_id] += _prior_question_had_explanation
user_all_count[_user_id][0] += 1
user_all_count[_user_id][1] += int(_answered_correctly == 1)
 except KeyError:
prior_question_had_explanation_count[_user_id] = _prior_question_had_explanation
user_all_count[_user_id] = [1, int(_answered_correctly == 1)]
try:
if len(user_correct_window_200[_user_id])== 201:
user_correct_window_200[_user_id].pop(0)
 prior_question_elapsed_time_window_dict[_user_id].pop(0)
 prior_question_had_explanation_window_dict[_user_id].pop(0)
 timediff_window_dict[_user_id].pop(0)
 user_correct_window_200[_user_id].append(int(_answered_correctly == 1))
prior_question_elapsed_time_window_dict[_user_id].append(_prior_question_elapsed_time)
 prior_question_had_explanation_window_dict[_user_id].append(_prior_question_had_explanation)
 timediff_window_dict[_user_id].append(_timediff)
 except KeyError:
user_correct_window_200[_user_id] = [int(_answered_correctly == 1)]
prior_question_elapsed_time_window_dict[_user_id] = [_prior_question_elapsed_time]
prior_question_had_explanation_window_dict[_user_id] = [_prior_question_had_explanation]
timediff_window_dict[_user_id] = [_timediff]<load_pretrained>
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.25,random_state = 123 )
if config.validaten_flg:
target_df = pd.read_pickle('.. /input/riiid-cross-validation-files/cv1_valid.pickle')
 iter_test = Iter_Valid(target_df, max_user=1000)
 predicted = []
def set_predict(df):
predicted.append(df)
 user_agg_feats_v = user_agg_feats_v[:10000]
last_timestamp_dic = {k: last_timestamp_dic[k] for k in user_agg_feats_v[:, 0]}
user_correct_window_200 = {k: user_correct_window_200[k] for k in user_agg_feats_v[:, 0]}
user_all_count = {k: user_all_count[k] for k in user_agg_feats_v[:, 0]}
prior_question_had_explanation_count = {k: prior_question_had_explanation_count[k] for k in user_agg_feats_v[:, 0]}
user_correct_window_200 = {k: user_correct_window_200[k] for k in user_agg_feats_v[:, 0]}
prior_question_elapsed_time_window_dict = {k: prior_question_elapsed_time_window_dict[k] for k in user_agg_feats_v[:, 0]}
prior_question_had_explanation_window_dict = {k: prior_question_had_explanation_window_dict[k] for k in user_agg_feats_v[:, 0]}
timediff_window_dict = {k: timediff_window_dict[k] for k in user_agg_feats_v[:, 0]}
else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<set_options>
model = Sequential()
model.add(Conv2D(32,kernel_size =(3,3),activation = 'relu',input_shape =(28,28,1)))
 model.add(Conv2D(64,kernel_size =(3,3),activation = 'relu'))
model.add(Flatten())
 model.add(Dense(10,activation = 'softmax'))
print(psutil.virtual_memory().percent )<install_modules>
model.compile(optimizer = 'adam',loss = 'categorical_crossentropy',metrics = ['accuracy'] )
!pip install.. /input/lgbm-inference-db-full-data/pickle5-0.0.11/<import_modules>
history = model.fit(X_train,y_train,validation_data =(X_test,y_test),epochs = 20 )
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
import pickle5 as pickle
from numba import jit<categorify>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes
'.format(name, round(( time.time() - t0)/ 60, 2)) )<define_variables>
model.compile(optimizer = 'Adam' , loss = "categorical_crossentropy", metrics=["accuracy"] )
train_pickle = '.. /input/lgbm-inference-db-full-data/train_df.pickle'
question_file = '.. /input/lgbm-inference-db-full-data/question_features.csv'
ms_in_a_day = 8.64 * 10 ** 7
prior_question_elapsed_time_mean = 25439.41<compute_train_metric>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
left_asymptote = 0.25
@jit(nopython=True)
 def get_new_theta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return theta + learning_rate_theta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote))
@jit(nopython=True)
 def learning_rate_theta(nb_answers):
return max(0.3 /(1 + 0.01 * nb_answers), 0.04)
 @jit(nopython=True)
 def probability_of_good_answer(theta, beta, left_asymptote):
return left_asymptote +(1 - left_asymptote)* sigmoid(theta - beta)
 @jit(nopython=True)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))<categorify>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 batch_size = 256
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = 30, validation_data =(X_test,y_test),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
def calc_user_feats_test(df, bundle_count, temp_values):
attempt_no_array = np.zeros(len(df), dtype=np.int32)
 last_lecture_time_array = np.zeros(len(df), dtype=np.float64)
 last_incorrect_time_array = np.zeros(len(df), dtype=np.float64)
 acsu = np.zeros(len(df), dtype=np.int32)
 acsu_part = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 cu_part = np.zeros(len(df), dtype=np.int32)
 tu_part = np.zeros(len(df), dtype=np.int32)
 lag_time_array = np.zeros(len(df), dtype=np.int64)
 wait_time_array = np.zeros(len(df), dtype=np.float64)
 theta_array = np.zeros(len(df), dtype=np.float32)
 beta_array = np.zeros(len(df), dtype=np.float32)
 difficulty_correct_array = np.zeros(len(df), dtype=np.float64)
 difficulty_incorrect_array = np.zeros(len(df), dtype=np.float64)
 feature_cols = ['user_id', 'prior_question_elapsed_time', 'timestamp',
'content_id', 'content_type_id', 'part', 'bundle_id',
'mean_content_accuracy_sm']
for cnt, row in enumerate(df[feature_cols].values):
if row[2] == 0:
lag_time_array[cnt] = 0
prior_question_lag_time[row[0]] = np.nan
wait_time_array[cnt] = np.nan
elif row[2] == user_last_timestamp[row[0]]:
wait_time_array[cnt] = temp_values[5]
lag_time_array[cnt] = row[2] - user_last_timestamp_traceback[row[0]]
else:
lag_time_array[cnt] = row[2] - user_last_timestamp[row[0]]
if(lag_time_array[cnt] == 0)|(lag_time_array[cnt] == row[2])|(len(prior_bundle_count[row[0]])!= 2):
wait_time_array[cnt] = np.nan
else:
wait_time_array[cnt] = prior_question_lag_time[row[0]] - prior_bundle_count[row[0]][1] * row[1]
user_last_timestamp_traceback[row[0]] = user_last_timestamp[
row[0]]
user_last_timestamp[row[0]] = row[2]
if row[4] == 1:
last_lecture_time[row[0]] = row[3]
else:
if row[6] in bundles:
if len(prior_bundle_count[row[0]])== 2:
if row[6] == prior_bundle_count[row[0]][0]:
bundle_count += 1
bundle_flg = True
save_temp_value_flg = False
else:
bundle_count = 1
bundle_flg = False
save_temp_value_flg = True
else:
bundle_count = 1
bundle_flg = False
save_temp_value_flg = True
else:
bundle_count = 1
bundle_flg = False
save_temp_value_flg = False
prior_question_lag_time[row[0]] = lag_time_array[cnt]
if save_temp_value_flg:
temp_values[0] = answered_correctly_sum_user_dict['total'][row[0]]
temp_values[1] = answered_correctly_sum_user_dict[int(row[5])][row[0]]
temp_values[2] = question_count_dict['total'][row[0]]
temp_values[3] = question_count_dict[int(row[5])][row[0]]
temp_values[4] = last_incorrect_time[row[0]]
temp_values[5] = wait_time_array[cnt]
temp_values[6] = beta_dict[row[3]]
temp_values[7] = theta_dict[row[0]]
temp_values[8] = difficulty_dict[row[0]]['correct']
temp_values[9] = difficulty_dict[row[0]]['incorrect']
if bundle_flg:
acsu[cnt] = temp_values[0]
cu[cnt] = temp_values[2]
difficulty_correct_array[cnt] = temp_values[8]
difficulty_incorrect_array[cnt] = temp_values[9]
acsu_part[cnt] = temp_values[1]
cu_part[cnt] = temp_values[3]
theta_array[cnt] = temp_values[7]
beta_array[cnt] = temp_values[6]
if row[2] == 0:
last_incorrect_time_array[cnt] = np.nan
else:
last_incorrect_time_array[cnt] = row[2] - temp_values[4]
else:
acsu[cnt] = answered_correctly_sum_user_dict['total'][row[0]]
cu[cnt] = question_count_dict['total'][row[0]]
difficulty_correct_array[cnt] = difficulty_dict[row[0]]['correct']
difficulty_incorrect_array[cnt] = difficulty_dict[row[0]]['incorrect']
acsu_part[cnt] = answered_correctly_sum_user_dict[int(row[5])][row[0]]
cu_part[cnt] = question_count_dict[int(row[5])][row[0]]
if row[2] == 0:
last_incorrect_time_array[cnt] = np.nan
else:
last_incorrect_time_array[cnt] = row[2] - last_incorrect_time[row[0]]
beta_array[cnt] = beta_dict[row[3]]
theta_array[cnt] = theta_dict[row[0]]
tu_part[cnt] = user_time_dict[int(row[5])][row[0]]
if bundle_count == 1:
attempt_dict[row[0]][row[6]] += 1
attempt_no_array[cnt] = attempt_dict[row[0]][row[6]]
if last_lecture_time[row[0]] == 0:
last_lecture_time_array[cnt] = np.nan
else:
last_lecture_time_array[cnt] = row[2] - last_lecture_time[row[0]]
if np.isnan(row[1]):
user_time_dict[int(row[5])][row[0]] += 0
else:
user_time_dict[int(row[5])][row[0]] += row[1]
prior_bundle_count[row[0]] =(row[6], bundle_count)
 df['attempt_no'] = attempt_no_array
df['last_lecture_time'] = last_lecture_time_array
df['last_incorrect_time'] = last_incorrect_time_array
df['lag_time'] = lag_time_array
df['prior_question_wait_time'] = wait_time_array
df['theta'] = theta_array
df['beta'] = beta_array
user_feats_df = pd.DataFrame({'answered_correctly_sum_user': acsu, 'answered_count': cu,
'answered_correctly_sum_user_part': acsu_part, 'answered_count_part': cu_part,
'total_time_spent_user_part': tu_part,
'difficulty_correct_count': difficulty_correct_array,
'difficulty_incorrect_count': difficulty_incorrect_array
})
 user_feats_df['mean_user_accuracy'] = user_feats_df['answered_correctly_sum_user'] / user_feats_df['answered_count']
user_feats_df['mean_user_accuracy_part'] = user_feats_df['answered_correctly_sum_user_part'] / user_feats_df[
'answered_count_part']
user_feats_df['mean_user_spent_time_part'] = user_feats_df['total_time_spent_user_part'] / user_feats_df[
'answered_count_part']
user_feats_df.loc[user_feats_df['answered_count_part'] == 0, 'mean_user_spent_time_part'] = np.nan
user_feats_df['difficulty_correct'] = user_feats_df['difficulty_correct_count'] / user_feats_df['answered_correctly_sum_user']
user_feats_df['difficulty_incorrect'] = user_feats_df['difficulty_incorrect_count'] / \ 
(user_feats_df['answered_count'] - user_feats_df['answered_correctly_sum_user'])
 user_feats_df['difficulty_diff'] = user_feats_df['difficulty_correct'] - user_feats_df['difficulty_incorrect']
user_feats_df.drop(columns=['difficulty_correct_count', 'difficulty_incorrect_count', 'difficulty_correct'], inplace=True)
 feats_cols = user_feats_df.columns
for col in feats_cols:
df[col] = user_feats_df[col].values
df['hmean_user_content_accuracy'] = 2 *(df['mean_user_accuracy'] * df['mean_content_accuracy_sm'])/ \ 
(df['mean_user_accuracy'] + df['mean_content_accuracy_sm'])
 df.loc[df['prior_question_wait_time'] < 0, 'prior_question_wait_time'] = 0
return df, bundle_count, temp_values 
<define_variables>
predictions = model.predict_classes(test, verbose=1 )
def update_user_feats(df):
bundle_count = 1
for row in df[['user_id', 'answered_correctly', 'content_type_id', 'timestamp',
'part', 'content_id', 'answered_count', 'mean_content_accuracy_sm']].values:
if row[2] == 0:
answered_correctly_sum_user_dict['total'][row[0]] += row[1]
answered_correctly_sum_user_dict[int(row[4])][row[0]] += row[1]
question_count_dict['total'][row[0]] += 1
question_count_dict[int(row[4])][row[0]] += 1
theta = theta_dict[row[0]]
beta = beta_dict[row[5]]
theta_dict[row[0]] = get_new_theta(row[1], beta, left_asymptote, theta, row[6])
 if row[1] == 0:
last_incorrect_time[row[0]] = row[3]
difficulty_dict[row[0]]['incorrect'] += row[7]
else:
difficulty_dict[row[0]]['correct'] += row[7]<load_pretrained>
prediction = pd.DataFrame({"ImageId":list(range(1,len(predictions)+1)) ,"Label":predictions} )
with open(train_pickle, 'rb')as file:
df = pickle.load(file )<categorify>
prediction.to_csv('kaggle_submission.csv',index=False,header=True)
 prediction
def multi_level_dict() :
return defaultdict(int)
 attempt_dict = defaultdict(multi_level_dict )<categorify>
model = Sequential()
model.add(Conv2D(64,(3,3),activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(64,(3,3),activation='relu'))
model.add(MaxPooling2D(2,2))
model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Conv2D(128,(3,3),activation='relu'))
model.add(Conv2D(128,(3,3),activation='relu'))
model.add(MaxPooling2D(2,2))
model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(128,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(64,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(10,activation='softmax'))
def multi_level_float_dict() :
return defaultdict(float )<data_type_conversions>
model.compile(RMSprop(lr=0.001,rho=0.9),loss='categorical_crossentropy',metrics=['accuracy'] )
with timer("counting"):
keys = np.sort(df['user_id'].unique())
 total = len(keys)
 user_bundle = df.groupby('user_id')['bundle_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = df.groupby(['user_id', 'bundle_id'])['bundle_id'].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array)
 for user_id, bundle, attempt in tqdm(zip(keys, user_bundle, user_attempts), total=total):
attempt_dict[user_id] = defaultdict(int, zip(bundle, attempt))
del user_bundle, user_attempts, df, bundle, attempt
gc.collect()<load_from_csv>
train_datagen = ImageDataGenerator(rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=False,
fill_mode='nearest')
 train_datagen.fit(X_train)
 train_generator = train_datagen.flow(X_train,y_train,batch_size=128)
<load_from_csv><EOS> 
earlystop = EarlyStopping(monitor='val_loss',patience=2,verbose=1)
 learning_reduce = ReduceLROnPlateau(patience=2,monitor="val_acc",verbose=1,min_lr=0.00001,factor=0.5)
 callbacks = [learning_reduce]
history = model.fit_generator(train_generator,epochs=30,verbose=1,validation_data=(X_test,y_test),callbacks=callbacks )