Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import random
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.utils import np_utils
gbm = lgb.Booster(model_file='.. /input/lightgbmmodel/model.txt' )<load_from_disk>
( X_train, y_train),(X_test, y_test)= mnist.load_data()
print("X_train shape", X_train.shape)
 print("y_train shape", y_train.shape)
 print("X_test shape", X_test.shape)
 print("y_test shape", y_test.shape )
json_file = ".. /input/lightgbm-features-dicts/lightgbm_features_dicts.json"
f = open(json_file)
 features_dicts = json.load(f )<drop_column>
test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv', delimiter = ',', header = 0, usecols = [x for x in range(0, 784)] )
del f<data_type_conversions>
X_train = X_train.reshape(60000, 784)
 X_test = X_test.reshape(10000, 784)
 X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train /= 255
X_test /= 255
print("Training matrix shape", X_train.shape)
 print("Testing matrix shape", X_test.shape )
def change_type(temp):
result = defaultdict(int)
 for k,v in temp.items() :
key = float(k)
 value = int(float(v))
result[key] =value
return result
answered_correctly_u_count = change_type(answered_correctly_u_count_)
 answered_correctly_u_sum = change_type(answered_correctly_u_sum_)
 elapsed_time_u_sum = change_type(elapsed_time_u_sum_)
 explanation_u_sum = change_type(explanation_u_sum_)
 answered_correctly_q_count = change_type(answered_correctly_q_count_)
 answered_correctly_q_sum = change_type(answered_correctly_q_sum_)
 elapsed_time_q_sum = change_type(elapsed_time_q_sum_)
 explanation_q_sum = change_type(explanation_q_sum_)
 del answered_correctly_u_count_
del answered_correctly_u_sum_
del elapsed_time_u_sum_
del explanation_u_sum_
del answered_correctly_q_count_
del answered_correctly_q_sum_
del elapsed_time_q_sum_
del explanation_q_sum_<categorify>
no_classes = 10
Y_train = np_utils.to_categorical(y_train, no_classes)
 Y_test = np_utils.to_categorical(y_test, no_classes )
def change_answered_correctly_uq_type(t):
result = defaultdict(lambda: defaultdict(int))
for k,v in t.items() :
key = float(k)
 temp = defaultdict(int)
 for k1,v1 in v.items() :
k1 = float(k1)
 temp[k1] = v1
result[key] = temp
return result
answered_correctly_uq = change_answered_correctly_uq_type(answered_correctly_uq_)
 del answered_correctly_uq_<categorify>
model = Sequential()
def changetolist(t):
result = defaultdict(list)
 for k,v in t.items() :
key = float(k)
 value = list(v)
 result[key] = value
return result
timestamp_u = changetolist(timestamp_u_)
 timestamp_u_incorrect = changetolist(timestamp_u_incorrect_)
 del timestamp_u_
del timestamp_u_incorrect_<define_variables>
model.add(Dense(512, input_shape=(784,)) )
TARGET = 'answered_correctly'
FEATURES = ['answered_correctly_u_avg', 'explanation_u_avg', 'elapsed_time_u_avg',
'answered_correctly_q_avg', 'explanation_q_avg', 'elapsed_time_q_avg',
'answered_correctly_uq_count', 'timestamp_u_recency_1',
'timestamp_u_recency_2', 'timestamp_u_recency_3',
'timestamp_u_incorrect_recency']
prior_question_elapsed_time_mean = 25423.810042960366
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df[TARGET] = 0
test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)
 test_df[TARGET] = gbm.predict(test_df[FEATURES],num_iteration=gbm.best_iteration)
 set_predict(test_df[['row_id', TARGET]])
 print('Job Done' )<import_modules>
model.add(Activation('relu'))
import riiideducation
import numpy as np
import pandas as pd
from tqdm import tqdm<compute_test_metric>
model.add(Dropout(0.2))
def get_new_theta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return theta + learning_rate_theta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote)
)
 def get_new_beta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return beta - learning_rate_beta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote)
)
 def learning_rate_theta(nb_answers):
return max(0.3 /(1 + 0.01 * nb_answers), 0.04)
 def learning_rate_beta(nb_answers):
return 1 /(1 + 0.05 * nb_answers)
 def probability_of_good_answer(theta, beta, left_asymptote):
return left_asymptote +(1 - left_asymptote)* sigmoid(theta - beta)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))<find_best_params>
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.2))
def estimate_parameters(answers_df, granularity_feature_name='content_id'):
item_parameters = {
granularity_feature_value: {"beta": 0, "nb_answers": 0}
for granularity_feature_value in np.unique(answers_df[granularity_feature_name])
 }
student_parameters = {
student_id: {"theta": 0, "nb_answers": 0}
for student_id in np.unique(answers_df.student_id)
 }
print("Parameter estimation is starting...")
 for student_id, item_id, left_asymptote, answered_correctly in tqdm(
 zip(answers_df.student_id.values, answers_df[granularity_feature_name].values, answers_df.left_asymptote.values, answers_df.answered_correctly.values)
):
theta = student_parameters[student_id]["theta"]
beta = item_parameters[item_id]["beta"]
item_parameters[item_id]["beta"] = get_new_beta(
 answered_correctly, beta, left_asymptote, theta, item_parameters[item_id]["nb_answers"], 
)
 student_parameters[student_id]["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta, student_parameters[student_id]["nb_answers"], 
)
 item_parameters[item_id]["nb_answers"] += 1
student_parameters[student_id]["nb_answers"] += 1
print(f"Theta & beta estimations on {granularity_feature_name} are completed.")
 return student_parameters, item_parameters<define_search_space>
model.add(Dense(10))
model.add(Activation('softmax'))
def update_parameters(answers_df, student_parameters, item_parameters, granularity_feature_name='content_id'):
for student_id, item_id, left_asymptote, answered_correctly in tqdm(zip(
 answers_df.student_id.values,
answers_df[granularity_feature_name].values,
answers_df.left_asymptote.values,
answers_df.answered_correctly.values)
):
if student_id not in student_parameters:
student_parameters[student_id] = {'theta': 0, 'nb_answers': 0}
if item_id not in item_parameters:
item_parameters[item_id] = {'beta': 0, 'nb_answers': 0}
theta = student_parameters[student_id]['theta']
beta = item_parameters[item_id]['beta']
student_parameters[student_id]['theta'] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta, student_parameters[student_id]['nb_answers'] 
)
 item_parameters[item_id]['beta'] = get_new_beta(
 answered_correctly, beta, left_asymptote, theta, item_parameters[item_id]['nb_answers'] 
)
 student_parameters[student_id]['nb_answers'] += 1
item_parameters[item_id]['nb_answers'] += 1
return student_parameters, item_parameters<compute_train_metric>
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] )
def estimate_probas(test_df, student_parameters, item_parameters, granularity_feature_name='content_id'):
probability_of_success_list = []
for student_id, item_id, left_asymptote in tqdm(
 zip(test_df.student_id.values, test_df[granularity_feature_name].values, test_df.left_asymptote.values)
):
theta = student_parameters[student_id]['theta'] if student_id in student_parameters else 0
beta = item_parameters[item_id]['beta'] if item_id in item_parameters else 0
probability_of_success_list.append(probability_of_good_answer(theta, beta, left_asymptote))
return probability_of_success_list<define_variables>
history = model.fit(X_train, Y_train,
batch_size=128, epochs=10,
verbose=1 )
compute_estimations = False
nb_rows_training = None<load_from_csv>
score = model.evaluate(X_test, Y_test)
 print('Test accuracy:', score[1] )
if compute_estimations:
training = pd.read_csv(
 filepath_or_buffer="/kaggle/input/riiid-test-answer-prediction/train.csv",
usecols=["content_id", "user_id", "answered_correctly"],
dtype={'answered_correctly': "int8"},
nrows=nb_rows_training 
)
 training.rename(columns={'user_id': 'student_id'}, inplace=True)
 training = training[training.answered_correctly != -1]
training['left_asymptote'] = 1/4
print(f"Dataset of shape {training.shape}")
 print(f"Columns are {list(training.columns)}")
 student_parameters, item_parameters = estimate_parameters(training)
 else:
student_data = pd.read_csv('.. /input/thetas-20201217/thetas_20201217.csv', index_col='student_id')
 student_parameters = student_data.to_dict('index')
 print(f"Successfully read student parameter file and converted to dict.")
 content_data = pd.read_csv('.. /input/betas-content-id-20201217/betas_content_id_20201217.csv', index_col='content_id')
 item_parameters = content_data.to_dict('index')
 print(f"Successfully read item parameter file and converted to dict." )<categorify>
results = model.predict(test_data )
def format_test_df(test_df):
test_copy = test_df.copy()
test_copy = test_copy[test_copy['content_type_id'] == 0]
test_copy['left_asymptote'] = 1/4
test_copy = test_copy.rename(columns={'user_id': 'student_id'})
 return test_copy<predict_on_test>
results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )
<compute_test_metric><EOS> 
predicted_classes = model.predict_classes(X_test)
 correct_indices = np.nonzero(predicted_classes == y_test)[0]
incorrect_indices = np.nonzero(predicted_classes != y_test)[0]