Kaggle competition: Digit Recognizer
def update_data(prior_group_answers_correct):
global tmp_data
arr = np.array(ast.literal_eval(prior_group_answers_correct))
for i, line in enumerate(tmp_data):
user_id = line[cols['user_id']]
content_type_id = line[cols['content_type_id']]
content_id = line[cols['content_id']]
timestamp = line[cols['timestamp']]
task_container_id = line[cols['task_container_id']]
explan = line[cols['prior_question_had_explanation']]
if isinstance(explan, pd._libs.missing.NAType)or explan == explan:
explan = 0
elapsed = line[cols['prior_question_elapsed_time']]
if isinstance(elapsed, pd._libs.missing.NAType)or elapsed == elapsed:
elapsed = 0
answered_correctly = arr[i]
user_arr = groups[user_id]
user_arr.insert(len(user_arr), answered_correctly)
 groups[user_id] = user_arr
if content_type_id == False:
if user_info.get(user_id, -1)== -1:
add_user(user_id)
 update_user(user_id, explan, elapsed, content_id, answered_correctly, timestamp)
 tmp_data = []<define_variables>
train=pd.read_csv("/kaggle/input/digit-recognizer/train.csv" )
tmp_data = []<define_variables>
test=pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
def preprocess_1(chunk):
data_1 = chunk.values
out = np.zeros(( data_1.shape[0], len(features)))
 batch_counts = data_1[:, [cols["user_id"],cols["content_type_id"]]]
batch_counts = Counter(batch_counts[batch_counts[:, 1] == False][:, 0])
 global tmp_data
for i in range(data_1.shape[0]):
user_id, content_type_id, content_id ,prior_group_answers_correct, timestamp, task_container_id, explan, elapsed = get_meta_data(data_1[i])
 part = parts.get(content_id, -1)
 task_count = batch_counts[user_id]
if prior_group_answers_correct == prior_group_answers_correct and prior_group_answers_correct != '[]':
update_data(prior_group_answers_correct)
 tmp_data.append(data_1[i].tolist())
 if content_type_id:
update_lec_data(user_id, content_id)
 if user_info.get(user_id, -1)== -1:
add_user(user_id)
 user_info[user_id]["interaction_n"] += 1
_ = non_lag_update(user_id, timestamp, elapsed, explan, content_type_id)
 diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5 = _
if not content_type_id:
if not isinstance(elapsed, pd._libs.missing.NAType)and elapsed == elapsed:
user_info[user_id]["el_sum"] += elapsed
else:
elapsed = 0
if isinstance(explan, pd._libs.missing.NAType)or explan != explan:
explan = 0
if user_info[user_id]["count_2"] != 0:
last_part = user_info[user_id]["last_part"]
user_info[user_id]["part_et"][last_part-1] += elapsed
user_info[user_id]["part_explan"][last_part-1] += explan
user_info[user_id]["last_part"] = part
user_info[user_id]["part_count_2"][part-1] += 1
user_info[user_id]["count_2"] += 1
out[i, features_dict['content_id']] = content_id
out[i, features_dict['task_container_id']] = data_1[i, cols['task_container_id']]
out[i, features_dict['prior_question_elapsed_time']] = elapsed
out[i, features_dict['el_avg']] =(user_info[user_id]["el_sum"]/(user_info[user_id]["count_2"])) /1000
out[i, features_dict['prior_question_elapsed_time_u_part_avg']] =(user_info[user_id]["part_et"][part-1])/(user_info[user_id]["part_count_2"][part-1])
 out[i, features_dict['prior_question_had_explanation']] = explan
out[i, features_dict['prior_question_had_explanation_u_part_avg']] = user_info[user_id]["part_explan"][part-1]/(user_info[user_id]["part_count_2"][part-1]+1)
 out[i, features_dict['bundle_id']:] = np.array(list(questions.get(content_id ).values()))
out[i, features_dict['que_count_user']] = user_info.get(user_id, {} ).get('count',0)
 out[i, features_dict['question_repeated']] = repeated_que_count.get(user_id, {} ).get(content_id, 0)+ 1
m = user_info.get(user_id, {} ).get('mean_acc', 0)
 if m == 0:
out[i, features_dict['user_mean']] = 0.55
else:
out[i, features_dict['user_mean']] = m
out[i, features_dict['opp_mean']] = 1 - out[i, features_dict['user_mean']]
out[i, features_dict['last_lecture']] = user_info.get(user_id, {} ).get('last_lec',0)
 out[i, features_dict['time_diff']] = diff_timestamp_1
out[i, features_dict['time_diff1']] = diff_timestamp_2
out[i, features_dict['time_diff2']] = diff_timestamp_3
out[i, features_dict['time_diff3']] = diff_timestamp_4
out[i, features_dict['time_diff4']] = diff_timestamp_5
out[i, features_dict['timestamp']] = timestamp/8.64e+7
out[i, features_dict['correct_recency']] =(timestamp - user_info[user_id]["recent_corr"])/8.64e+7
kk = user_info[user_id]["priors_5"][-5:]
if len(kk)!= 0:
out[i, features_dict['rolling_mean_5_prior_question']] = np.array(kk ).mean()
out[i, features_dict['ts_diff_shifted']] = user_info[user_id]["ts_diff_shifted"]
out[i, features_dict['ts_diff_shifted_2']] = user_info[user_id]["ts_diff_shifted_2"]
out[i, features_dict['container_mean']] = containers_mean[task_container_id]
out[i, features_dict['lecs_per']] = user_info[user_id]["lecs_n"]/user_info[user_id]["interaction_n"]*100
out[i, features_dict['sessions']] = user_info[user_id]["sessions"]
out[i, features_dict['session_count']] = user_info[user_id]["session_count"]
if user_info[user_id]["count"] != 0:
out[i, features_dict['prior_question_had_explanation_ratio']] = user_info[user_id]["had_exp"]/user_info[user_id]["count"]
if user_info[user_id]["sessions"] != 0:
out[i, features_dict['mean_pause']] = user_info[user_id]["sum_pauses"]/user_info[user_id]["sessions"]
nn = user_info.get(user_id, {} ).get('easy_ct',0)
 if nn != 0:
out[i, features_dict['easy_ratio']] = user_info.get(user_id, {} ).get('easy_cr',0)/nn
out[i, features_dict['easy_ratio_opp']] = 1 - out[i, features_dict['easy_ratio']]
nn = user_info.get(user_id, {} ).get('hard_ct',0)
 if nn != 0:
out[i, features_dict['hard_ratio']] = user_info.get(user_id, {} ).get('hard_cr',0)/nn
out[i, features_dict['hard_ratio_opp']] = 1 - out[i, features_dict['hard_ratio']]
nn = user_info.get(user_id, {} ).get('part_count',[])
 if nn != []:
part = int(out[i, features_dict['part']] - 1)
 ct = nn[part]
cr = user_info[user_id]['part_corr'][part]
if ct != 0:
out[i, features_dict['part_mean']] = cr/ct
if groups.get(user_id, -1)!= -1 and groups[user_id] != []:
last_arr = np.array(groups[user_id][-40:])
 out[i, features_dict['rolling_mean']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]
last_arr = last_arr[last_arr != -1]
out[i, features_dict['ewm_mean_10']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]
out[i, features_dict['rolling_mean_10']] = np.array(last_arr[-10:] ).mean()
out[i, features_dict['rolling_mean_15']] = np.array(last_arr[-15:] ).mean()
out[i, features_dict['rolling_mean_5']] = np.array(last_arr[-5:] ).mean()
out[i, features_dict['prior_part_mean']] = prior_part_mean_dict[int(out[i, features_dict['part']])]
k = que_2_k[content_id]
out[i, features_dict['k']] = k
if user_info[user_id]["k_count"][k] != 0:
out[i, features_dict['k_acc']] = user_info[user_id]["k_corr"][k]/user_info[user_id]["k_count"][k]
out[i, features_dict['wut']] = user_info[user_id]["ts_diff_shifted"] - elapsed*task_count
if out[i, features_dict['wut']] < 0:
out[i, features_dict['wut']] = 0
usr = watched_tags.get(str(user_id), -1)
 if usr != -1:
nn = 0
for k in range(6):
nn += usr.get(str(int(questions1[content_id]["tags"+str(k+1)])) , 0)
 out[i, features_dict['watched']] = nn
return out 
<define_variables>
label=train["label"].values
model_lgbm = lgb.Booster(model_file='.. /input/lgbm-test/lgb_classifier.txt')
 stack_lgbm = lgb.Booster(model_file='.. /input/lgbm-test/lgb_stack.txt' )<define_variables>
train.drop("label",axis=1,inplace=True )
vals = 0<split>
t=train.values
ttest=test.values
%%time
for(test_data,sample_prediction_df)in iter_test:
if not isinstance(vals, int):
if test_data.iloc[0].prior_group_answers_correct == test_data.iloc[0].prior_group_answers_correct:
past_vals = np.array(ast.literal_eval(test_data.iloc[0].prior_group_answers_correct))
past_answers = np.array(ast.literal_eval(test_data.iloc[0].prior_group_responses))
past_vals = np.concatenate(( vals, past_vals.reshape(len(past_vals),1)) , axis=1)
 past_vals = np.concatenate(( past_vals, past_answers.reshape(len(past_answers),1)) , axis=1)
 update_group_var(past_vals)
 vals = test_data[["user_id","content_id", "content_type_id", "task_container_id","timestamp","prior_question_elapsed_time"]].values
test_transform = preprocess_1(test_data)
 lgbm_predic = model_lgbm.predict(test_transform)
 st_predic = np.array(split_preds(vals))
test_data['answered_correctly'] =(lgbm_predic*0.85 + 1.15*st_predic)/2
env.predict(test_data.loc[test_data['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<categorify>
t=t.astype('float32')
 ttest=ttest.astype('float32' )
dicts_path = '/kaggle/input/agg-riiid/agg_riiid/'
user_content_id_agg = pd.read_pickle(dicts_path + 'user_content_id_agg.pkl.gzip')
 user_content_id_agg['count'] = user_content_id_agg['count'].astype('int16' )<data_type_conversions>
t/=255
ttest/=255
user_content_id_count_dict = user_content_id_agg['count'].astype('int16' ).to_dict(defaultdict(int))<set_options>
tl = keras.utils.to_categorical(label, 10)
del user_content_id_agg
gc.collect()<import_modules>
input_shape =(28, 28, 1 )
import numpy as np
import lightgbm as lgb
import pickle
import riiideducation
import joblib<define_variables>
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=input_shape))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(64,(3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128,(3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256,(3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
target = 'answered_correctly'<define_variables>
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'] )
features = [
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'user_correctness',
'user_correctness_window_10_mean',
'part',
'content_count',
'content_sum',
'content_id_correctness_total',
'repeated_times',
'user_count_questions',
'explanation_mean_user',
'timestamp',
'timestamp_diff_last',
'user_count_lectures',
'last_lecture_tag',
'prior_questions_mean_time',
'last_correct',
'task_container_id',
'first_tag',
'second_tag',
'repeated_times_tag',
'user_tag_correctness',
'timestamp_diff_last_window_10_mean',
'user_count_lectures_mean'
]<categorify>
np.random.seed(1234)
(x_train,x_test,y_train,y_test)= train_test_split(t,tl, train_size=0.75, random_state=1 )
dicts_path = '/kaggle/input/agg-riiid/agg_riiid/'
user_agg = pd.read_pickle(dicts_path + 'user_agg.pkl.gzip')
 user_sum_dict = user_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
gc.collect()
content_agg = pd.read_pickle(dicts_path + 'content_agg.pkl.gzip')
 content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del content_agg
gc.collect()
user_target_window_10 = pd.read_pickle(dicts_path + 'user_target_window_10.pkl.gzip')
 user_target_window_10_dict = user_target_window_10.astype('int8' ).to_dict(defaultdict(int))
del user_target_window_10
gc.collect()
user_target_next_out_window_10_dict = defaultdict(int)
 timestamp_diff_window_10 = pd.read_pickle(dicts_path + 'timestamp_diff_window_10.pkl.gzip')
 timestamp_diff_window_10_dict = timestamp_diff_window_10.astype('int32' ).to_dict(defaultdict(int))
del timestamp_diff_window_10
gc.collect()
timestamp_diff_next_window_10_dict = defaultdict(int)
 explanation_agg = pd.read_pickle(dicts_path + 'explanation_agg.pkl.gzip')
 explanation_sum_dict = explanation_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del explanation_agg
gc.collect()
user_lectures_agg = pd.read_pickle(dicts_path + 'user_lectures_agg.pkl.gzip')
 user_lectures_count_dict = user_lectures_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
del user_lectures_agg
gc.collect()
last_lecture_agg = pd.read_pickle(dicts_path + 'last_lecture_agg.pkl.gzip')
 last_lecture_tag_dict = last_lecture_agg.astype('int16' ).to_dict(defaultdict(int))
del last_lecture_agg
gc.collect()
prior_questions_time_agg = pd.read_pickle(dicts_path + 'prior_questions_time_agg.pkl.gzip')
 prior_questions_time_sum_dict = prior_questions_time_agg['sum'].astype('int32' ).to_dict(defaultdict(float))
prior_questions_time_count_dict = prior_questions_time_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del prior_questions_time_agg
gc.collect()
last_correct_agg = pd.read_pickle(dicts_path + 'last_correct_agg.pkl.gzip')
 last_correct_dict = last_correct_agg.astype('int8' ).to_dict(defaultdict(int))
del last_correct_agg
gc.collect()
user_tag_agg = pd.read_pickle(dicts_path + 'user_tag_agg.pkl.gzip')
 user_tag_agg_sum_dict = user_tag_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_tag_agg_count_dict = user_tag_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_tag_agg
gc.collect()
last_timestamp_agg = pd.read_pickle(dicts_path + 'last_timestamp_agg.pkl.gzip')
 last_timestamp_dict = last_timestamp_agg.astype('int32' ).to_dict(defaultdict(int))
del last_timestamp_agg
gc.collect()<load_pretrained>
model.fit(x_train, y_train,
batch_size=100,
epochs=400,
verbose=2,
validation_data=(x_test, y_test))
model_path = '/kaggle/input/trained-model/'
file = model_path + 'trained_model.pkl'
model = pickle.load(open(file, 'rb'))
print('Trained LGB model was loaded!' )<load_from_csv>
y_pred=model.predict(ttest,verbose=0 )
home_path = '/kaggle/input/riiid-test-answer-prediction/'
questions_df = pd.read_csv(home_path + 'questions.csv',
usecols=[0, 3, 4],
dtype={'question_id': 'int16', 'part': 'int8'}
)<data_type_conversions>
sample=pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )
questions_df['tags'].fillna('92', inplace=True)
 questions_df['first_tag'] = questions_df['tags'].apply(lambda x: x.split() [0])
 questions_df['first_tag'] = questions_df['first_tag'].astype('int16')
 questions_df['second_tag'] = questions_df['tags'].apply(lambda x: x.split() [1] if len(x.split())> 1 else -1)
 questions_df['second_tag'] = questions_df['second_tag'].astype('int16')
 questions_df.drop('tags',axis=1,inplace=True )<load_from_csv>
pred = np.argmax(y_pred, axis = 1 )
lectures_df = pd.read_csv(home_path + 'lectures.csv',
dtype={'tag': 'int16', 'part': 'int8'} 
)
 type_of_dict = {'intention': 1, 'concept': 2, 'solving question': 3, 'starter': 4}
lectures_df['type_of'] = lectures_df['type_of'].map(type_of_dict )<import_modules>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),r],axis = 1 )
import random
from tqdm.notebook import tqdm
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader<load_pretrained>
submission.to_csv("mnist4.csv",index=False )