Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
warnings.filterwarnings('ignore' )
df = pd.get_dummies(df, columns = ["Title"])
 df = pd.get_dummies(df, columns = ["Embarked"])
 df = pd.get_dummies(df, columns = ["Pclass"])
 df = pd.get_dummies(df, columns = ["Age Group"] )<drop_column>
import pickle
import riiideducation
from tqdm import tqdm
import glob
import pandas as pd
import numpy as np
import plotly.graph_objects as go
train = df[:train.shape[0]]
test = df[train.shape[0]:].drop(["Survived"], axis = 1 )<train_on_grid>
metrics = [
'questions_part',
'questions_difficulty',
'questions_count',
'questions_good_answer_reaction_time',
'questions_bad_answer_reaction_time',
'questions_batch_size',
'questions_variance',
'questions_difficulty_given_prev_explanation',
'questions_difficulty_given_prev_no_explanation',
'questions_conditionnal_proba_cluster',
'lectures_count',
'student_general_elo_theta',
'student_general_elo_proba',
'student_part_elo_proba',
'student_tags_elo_theta',
'student_tags_elo_proba',
'student_part_score',
'timestamp_student_general_last_batch_average',
'timestamp_student_general_reactivity_rolling_19',
'timestamp_student_general_part_reactivity_average',
'timestamp_student_general_reactivity_good_answer_average',
'timestamp_last_delta_between_questions',
'timestamp_delta_between_questions_rolling_10',
'timestamp_delta_between_questions_rolling_5',
'timestamp_last_time_between_answers',
'student_general_good_answers_average',
'student_general_good_answers_rolling_2',
'student_general_good_answers_rolling_6',
'student_part_good_answers_rolling_6',
'questions_seen_with_explanation',
'questions_conditionnal_count',
'questions_conditionnal_proba',
'lectures_conditionnal_proba',
]
k_fold = StratifiedKFold(n_splits=5)
 X_train = train.drop(labels="Survived", axis=1)
 y_train = train["Survived"]
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 LG_classifier = LogisticRegression(random_state=0)
 SVC_classifier = SVC(kernel="rbf", random_state=0)
 KNN_classifier = KNeighborsClassifier()
NB_classifier = GaussianNB()
DT_classifier = DecisionTreeClassifier(criterion="entropy", random_state=0)
 RF_classifier = RandomForestClassifier(n_estimators=200, criterion="entropy", random_state=0)
 titanic_classifiers = [LG_classifier, SVC_classifier, KNN_classifier, NB_classifier, DT_classifier, RF_classifier]
classifier_dict = {
0: "Logistic Regression",
1: "Support Vector Classfication",
2: "K Nearest Neighbor Classification",
3: "Naive bayes Classifier",
4: "Decision Trees Classifier",
5: "Random Forest Classifier",
}
titanic_results = pd.DataFrame({'Model': [],'Mean Accuracy': [], "Standard Deviation": []})
 for i, classifier in enumerate(titanic_classifiers):
classifier_scores = cross_val_score(classifier, X_train, y_train, cv=k_fold, n_jobs=2, scoring="accuracy")
 titanic_results = titanic_results.append(pd.DataFrame({"Model":[classifier_dict[i]],
"Mean Accuracy": [classifier_scores.mean() ],
"Standard Deviation": [classifier_scores.std() ]}))<data_type_conversions>
with open('.. /input/lgb-training/model.pkl', 'rb')as f:
lgb = pickle.load(f )
print(titanic_results.to_string(index=False))<choose_model_class>
train = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', nrows = 1000000, index_col = 0)
 questions = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('.. /input/riiid-test-answer-prediction/lectures.csv' )
RF_classifier = RandomForestClassifier()
RF_paramgrid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,200,300],
"criterion": ["entropy"]}
RF_classifiergrid = GridSearchCV(RF_classifier, param_grid = RF_paramgrid, cv=k_fold, scoring="accuracy", n_jobs= -1, verbose=1)
 RF_classifiergrid.fit(X_train,y_train)
 RFC_optimum = RF_classifiergrid.best_estimator_
RF_classifiergrid.best_score_<define_variables>
questions_to_difficulty =(1-train[train.content_type_id == False].groupby('content_id')['answered_correctly'].mean() )
IDtest = passenger_ID[train.shape[0]:].reset_index(drop = True )<save_to_csv>
subtrain = train[train.content_type_id ==0]
subtrain['shift_elapse_time'] = subtrain['prior_question_elapsed_time'].shift(-1)
 good_answer_reaction_time = subtrain[subtrain.answered_correctly==True].dropna().groupby('content_id')['shift_elapse_time'].mean()
bad_answer_reaction_time = subtrain[subtrain.answered_correctly==False].dropna().groupby('content_id')['shift_elapse_time'].mean()
X_train = train.drop(labels="Survived", axis=1)
 y_train = train["Survived"]
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.fit_transform(test)
 RFC_optimum.fit(X_train, y_train)
 test_predictions = pd.Series(RFC_optimum.predict(X_test ).astype(int), name="Survived")
 titanic_results = pd.concat([IDtest, test_predictions], axis = 1)
 titanic_results.to_csv('submission.csv', index=False )<set_options>
( unique, counts)= np.unique(train.values[:,1], return_counts=True)
 qids = pd.Series(counts, index = unique ).sort_values(ascending=False)[:500]
qids.head()
warnings.filterwarnings('ignore')
 %matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams[ 'figure.figsize' ] = 8 , 6 
<load_from_csv>
qids = qids.index
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )<create_dataframe>
memory = 100
train_arr = train[['user_id', 'content_id', 'content_type_id', 'answered_correctly']].values
user_meta = {}
user_meta_count = {}
meta_proba_pos = {}
meta_count_pos = {}
meta_proba_neg = {}
meta_count_neg = {}
for i in tqdm(range(len(train))):
user_id, content_id, content_type_id, answered_correctly = train_arr[i]
if user_id not in user_meta.keys() :
user_meta = {}
user_meta_count = {}
user_meta[user_id] = {qid:0 for qid in qids}
user_meta_count[user_id] = {}
if content_type_id ==0:
A = content_id
X = np.array(list(user_meta[user_id].values()))
if A not in meta_proba_pos.keys() :
meta_proba_pos[A] = {i:0 for i in range(len(qids)) }
meta_count_pos[A] = {i:0 for i in range(len(qids)) }
meta_proba_neg[A] = {i:0 for i in range(len(qids)) }
meta_count_neg[A] = {i:0 for i in range(len(qids)) }
for B in np.where(X!=0)[0]:
if(X[B]==1):
if answered_correctly == 1:
meta_proba_pos[A][B] += 1
meta_count_pos[A][B] += 1
if(X[B]==-1):
if answered_correctly == 1:
meta_proba_neg[A][B] += 1
meta_count_neg[A][B] += 1
if A in qids:
for k,v in user_meta_count[user_id].copy().items() :
user_meta_count[user_id][k] = v-1
if v == 0:
del user_meta_count[user_id][k]
user_meta[user_id][k] = 0
user_meta[user_id][content_id] = 2*answered_correctly-1
if not A in user_meta_count[user_id].keys() :
user_meta_count[user_id][A] = memory
df = train.copy()<split>
with open('.. /input/riid-raw-samples/bad_answer_reaction_time', 'rb')as f:
bad_answer_reaction_time = pickle.load(f)
 with open('.. /input/riid-raw-samples/good_answer_reaction_time', 'rb')as f:
good_answer_reaction_time = pickle.load(f)
 with open('.. /input/riid-raw-samples/question_user_count', 'rb')as f:
question_user_count = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty', 'rb')as f:
questions_to_difficulty = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty', 'rb')as f:
questions_to_difficulty = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty_variance', 'rb')as f:
questions_to_difficulty_variance = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty_with_explanation', 'rb')as f:
questions_to_difficulty_with_explanation = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty_without_explanation', 'rb')as f:
questions_to_difficulty_without_explanation = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_count.pkl', 'rb')as f:
meta_count = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_count_neg.pkl', 'rb')as f:
meta_count_neg = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_proba.pkl', 'rb')as f:
meta_proba = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_proba_neg.pkl', 'rb')as f:
meta_proba_neg = pickle.load(f)
 with open('.. /input/riid-probabilities/question_position.pkl', 'rb')as f:
question_position = pickle.load(f)
 with open('.. /input/riid-lectures/lecture_position.pkl', 'rb')as f:
lecture_position = pickle.load(f)
 with open('.. /input/riid-lectures/meta_count_lect.pkl', 'rb')as f:
meta_count_lect = pickle.load(f)
 with open('.. /input/riid-lectures/meta_proba_lect.pkl', 'rb')as f:
meta_proba_lect = pickle.load(f)
 clust = pd.read_csv('.. /input/riid-cluster/test_cluster_4',index_col=0)['0']
with open('.. /input/riid-cache-questions/cache_questions.pck', 'rb')as f:
question_cache = pickle.load(f )
y = df.label.values.astype('int32')
 df = df[df.columns[1:]].values.astype('float32')
 X_train , X_test , y_train , y_test = train_test_split(df , y , test_size = 0.2 , random_state = 100 )<data_type_conversions>
proba_pos = {}
proba_neg = {}
THRESH_COUNT = 50
THRESH_PROBA = 0.01
for A,v in tqdm(meta_proba.items()):
proba_pos[A] = {}
proba_neg[A] = {}
for B in v.keys() :
if meta_count[A][B]>THRESH_COUNT:
proba = meta_proba[A][B]/meta_count[A][B]
improve = proba -(1-questions_to_difficulty[A])
 if improve > THRESH_PROBA:
proba_pos[A][question_position[B]] = improve
if meta_count_neg[A][B]>THRESH_COUNT:
proba = meta_proba_neg[A][B]/meta_count_neg[A][B]
decrease = proba -(1-questions_to_difficulty[A])
 if decrease < -THRESH_PROBA:
proba_neg[A][question_position[B]] = decrease
proba_lec = {}
thresh = 20
for k,v in tqdm(meta_proba_lect.items()):
proba_lec[k] = {}
for kk,vv in v.items() :
if meta_count_lect[k][kk]>thresh:
proba = meta_proba_lect[k][kk]/meta_count_lect[k][kk]
improve = proba -(1-questions_to_difficulty[k])
 if improve > 0.10:
proba_lec[k][lecture_position[kk]] = improve
test = test.values.astype('float32' )<load_from_csv>
all_prob = {}
for k,v in proba_pos.items() :
all_prob[k] = {}
for kk,vv in v.items() :
all_prob[k][(kk,1)] = vv
for k,v in proba_neg.items() :
if k not in all_prob.keys() :
all_prob[k] = {}
for kk,vv in v.items() :
all_prob[k][(kk,0)] = vv
all_q = set()
for k,v in all_prob.items() :
for kk in v.keys() :
all_q.add(kk[0])
 mprob1 = {}
for k,v in tqdm(all_prob.items()):
mprob1[k] = {}
kk = [e[0] for e in v.keys() ]
for elmt in all_q:
mprob1[k][elmt]=0
if elmt in kk:
if(elmt,1)in v.keys() :
mprob1[k][elmt]=v[(elmt,1)]
else:
mprob1[k][elmt]=v[(elmt,0)]
new = pd.read_csv('.. /input/digit-recognizer/train.csv' )<count_values>
mprob = pd.DataFrame(mprob1 ).T
mprob.head()
label_counter = new.label.value_counts()
print(label_counter )<feature_engineering>
kmean = KMeans(n_clusters=20)
 c = kmean.fit_predict(mprob )
X_train = X_train / 255
X_test = X_test / 255
test = test / 255<categorify>
ids = glob.glob('.. /input/riid-cache-6/content/drive/My Drive/riid/*')
 ids = [int(elmt.split('/')[-1])for elmt in ids]
y_train = keras.utils.to_categorical(y_train, 10)
 y_test = keras.utils.to_categorical(y_test, 10 )<import_modules>
questions_to_tag = questions[['question_id','tags']].set_index('question_id')[['tags']]['tags'].fillna('-1' ).apply(lambda x: [np.int64(elmt)for elmt in x.split(' ')] ).to_dict()
questions_to_parts = questions[['question_id','part']].set_index('question_id')['part'].to_dict()
lecture_to_type = lectures[['lecture_id','type_of']].set_index('lecture_id')['type_of'].to_dict()
lecture_to_part = lectures[['lecture_id','part']].set_index('lecture_id')['part'].to_dict()
lecture_to_tag = lectures[['lecture_id','tag']].set_index('lecture_id')['tag'].to_dict()
tags = []
for k,v in questions_to_tag.items() :
tags+= v
tags = {elmt:np.int32(0)for elmt in set(tags)}
parts = np.unique(list(questions_to_parts.values()))
question_list = list(questions_to_difficulty.keys() )
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau<choose_model_class>
def qscore(answer, difficulty):

 if answer>0:
return(2*answer - 1)*difficulty
else:
return(2*answer - 1)*(1-difficulty)
 def list_feature_average(n, vals):

 if len(vals)>n:
return np.mean(vals[-n:])
 else:
return -1
def make_feature_average(count, vals):

 if count>0:
return vals/count
else:
return -1
def calculate_time_deltas(cache, timestamp, prior_question_elapsed_time,content_type_id, previous_part, user_id, batch_size):
if timestamp != cache[user_id]['ts-1']:
if prior_question_elapsed_time != -1:
cache[user_id]['average_reactivities'].append(prior_question_elapsed_time)
 cache[user_id]['last_ts_list'].append(timestamp)
 if len(cache[user_id]['average_reactivities'])>20:
cache[user_id]['average_reactivities'].pop(0)
 if len(cache[user_id]['last_ts_list'])>20:
cache[user_id]['last_ts_list'].pop(0)
 if previous_part in cache[user_id]['part_average_reactivity_tot'].keys() :
cache[user_id]['part_average_reactivity_tot'][previous_part]+=prior_question_elapsed_time
else:
if previous_part != -1:
cache[user_id]['part_average_reactivity_tot'][previous_part] = prior_question_elapsed_time
if len(cache[user_id]['last_ts_list'])>3:
cache[user_id]['time_between_questions'].append(cache[user_id]['last_ts_list'][-2]-cache[user_id]['last_ts_list'][-3]-cache[user_id]['average_reactivities'][-1]*cache[user_id]['batch_size_list'])
 if len(cache[user_id]['time_between_questions'])>20:
cache[user_id]['time_between_questions'].pop(0)
 if(cache[user_id]['last_delta_between_questions'] != -1)&(prior_question_elapsed_time != -1):
if cache[user_id]['average_delta_between_questions'] > 3600:
cache[user_id]['hour_count'] +=1
if cache[user_id]['average_delta_between_questions'] > 10000:
cache[user_id]['day_count'] +=1
if cache[user_id]['average_delta_between_questions'] < 3600:
cache[user_id]['average_delta_between_questions'] += cache[user_id]['last_delta_between_questions'] - prior_question_elapsed_time
if cache[user_id]['ts-1'] != -1:
if content_type_id == 0:
cache[user_id]['last_delta_between_questions'] =(timestamp - cache[user_id]['ts-1'])/batch_size
else:
cache[user_id]['average_time_on_lecture'] +=(timestamp - cache[user_id]['ts-1'])/batch_size
return cache
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='relu', padding='same',input_shape=(28 , 28 , 1)))
 model.add(Conv2D(32,(3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(64,(3, 3), activation='relu', padding='same'))
model.add(Conv2D(64,(3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(100, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))<choose_model_class>
def get_new_theta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return theta + learning_rate_theta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote)
)
 def get_new_beta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return beta - learning_rate_beta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote)
)
 def learning_rate_theta(nb_answers):
return max(0.3 /(1 + 0.01 * nb_answers), 0.04)
 def learning_rate_beta(nb_answers):
return 1 /(1 + 0.05 * nb_answers)
 def probability_of_good_answer(theta, beta, left_asymptote):
return left_asymptote +(1 - left_asymptote)* sigmoid(theta - beta)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))
model.compile(optimizer = RMSprop(lr=0.001),loss='categorical_crossentropy', metrics=['accuracy'] )<init_hyperparams>
def create_cache(cache, user_id, ts, ids):
if user_id in ids:
with open(f'.. /input/riid-cache-6/content/drive/My Drive/riid/{user_id}', 'rb')as f:
id_cache = pickle.load(f)[user_id]
cache[user_id] = id_cache
else:
cache[user_id] = {}
cache[user_id]['previous_content_type_id'] = -1
cache[user_id]['previous_part'] = -1
cache[user_id]['previous_question'] = -1
cache[user_id]['questions_count'] = 0
cache[user_id]['part_lectures_count'] = {i:0 for i in range(1,8)}
cache[user_id]['lectures_part'] = {i:0 for i in range(1,8)}
cache[user_id]['part_count'] = {}
cache[user_id]['question_with_explanation'] = set()
cache[user_id]['question_seen'] = set()
cache[user_id]['first_bundle'] = -1
cache[user_id]['batch_size_list'] = -1
cache[user_id]['ts-1'] = -1
cache[user_id]['last_ts_part'] = {i:0 for i in range(1,8)}
cache[user_id]['last_delta_between_questions'] = -1
cache[user_id]['day_count'] = 0
cache[user_id]['hour_count'] = 0
cache[user_id]['last_ts_list'] = []
cache[user_id]['time_between_questions'] = []
cache[user_id]['average_reactivity_tot'] = 0
cache[user_id]['average_reaction_good_answer_tot'] = 0
cache[user_id]['average_reaction_bad_answer_tot'] = 0
cache[user_id]['average_delta_between_good_answer'] = 0
cache[user_id]['part_average_delta_between_good_answer'] = {}
cache[user_id]['average_reactivities'] = []
cache[user_id]['average_delta_between_questions'] = 0
cache[user_id]['average_time_on_lecture'] = -1
cache[user_id]['part_average_reactivity_tot'] = {}
cache[user_id]['last_questions'] = []
cache[user_id]['last_answers_correctly'] = []
cache[user_id]['last_answers_correctly_part'] = {i:[] for i in range(1,8)}
cache[user_id]['last_lectures'] = []
cache[user_id]['lectures_count'] = 0
cache[user_id]['general_good_answer'] = 0
cache[user_id]['part_good_answer'] = {}
cache[user_id]['part_score'] = {i:0 for i in range(1,8)}
cache[user_id]['student_parameters'] = {"theta": 0, "nb_answers": 0}
cache[user_id]['part_student_parameters'] = {i:{"theta": 0, "nb_answers": 0} for i in range(1,8)}
cache[user_id]['tag_student_parameters'] = {i:{"theta": 0, "nb_answers": 0} for i in tags.keys() }
cache[user_id]['tag_count'] = {}
cache[user_id]['tag_good_answers'] = {}
return cache
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.0,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
def update_cache(cache, arr, batch_size, question_cache):
left_asymptote = 1/4
row_id, timestamp, user_id, content_id, content_type_id, task_container_id, prior_question_elapsed_time, prior_question_had_explanation, _, _, answered_correctly,user_answer = arr
if not prior_question_elapsed_time:
prior_question_elapsed_time=-1
if not prior_question_had_explanation:
prior_question_had_explanation =-1
timestamp = int(float(timestamp)/1000)
 user_id = np.uint32(user_id)
 content_id = np.uint16(content_id)
 content_type_id = np.int8(content_type_id)
 task_container_id = np.uint32(task_container_id)
 prior_question_elapsed_time = int(float(prior_question_elapsed_time)/1000)
 prior_question_had_explanation = np.bool(prior_question_had_explanation)
 user_answer = np.int8(user_answer)
 answered_correctly = np.int8(answered_correctly)
 previous_content_type_id = cache[user_id]['previous_content_type_id']
part = -1
if content_type_id == 0:
if content_id not in question_cache.keys() :
question_cache[content_id] = {"beta": 0, "nb_answers": 0}
part = questions_to_parts[content_id]
tags = questions_to_tag[content_id]
difficulty = questions_to_difficulty[content_id]
count = question_user_count[content_id]
bad_answer_reaction = bad_answer_reaction_time[content_id]
good_answer_reaction = good_answer_reaction_time[content_id]
beta = question_cache[content_id]["beta"]
score = qscore(answered_correctly, difficulty)
 theta = cache[user_id]['student_parameters']["theta"]
theta_part = cache[user_id]['part_student_parameters'][part]["theta"]
for tag in tags:
theta_tag = cache[user_id]['tag_student_parameters'][tag]["theta"]
cache[user_id]['tag_student_parameters'][tag]["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta_tag, cache[user_id]['tag_student_parameters'][tag]["nb_answers"], 
)
 cache[user_id]['tag_student_parameters'][tag]["nb_answers"] += 1
cache[user_id]['student_parameters']["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta, cache[user_id]['student_parameters']["nb_answers"], 
)
 cache[user_id]['part_student_parameters'][part]["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta_part, cache[user_id]['part_student_parameters'][part]["nb_answers"], 
)
 cache[user_id]['student_parameters']["nb_answers"] += 1
cache[user_id]['part_student_parameters'][part]["nb_answers"] += 1
cache[user_id]['questions_count'] +=1
if part in cache[user_id]['part_count'].keys() :
cache[user_id]['part_count'][part] += 1
else:
cache[user_id]['part_count'][part] = 1
cache[user_id]['last_answers_correctly'].append(answered_correctly)
 cache[user_id]['last_answers_correctly_part'][part].append(answered_correctly)
 if len(cache[user_id]['last_answers_correctly'])>20:
cache[user_id]['last_answers_correctly'].pop(0)
 if len(cache[user_id]['last_answers_correctly_part'][part])>20:
cache[user_id]['last_answers_correctly_part'][part].pop(0)
 if answered_correctly == 1:
cache[user_id]['average_reaction_good_answer_tot'] += prior_question_elapsed_time
if part in cache[user_id]['part_good_answer'].keys() :
cache[user_id]['part_good_answer'][part]+=1
else:
cache[user_id]['part_good_answer'][part]=1
cache[user_id]['general_good_answer'] += 1
score = qscore(answered_correctly, difficulty)
 if answered_correctly == 0:
cache[user_id]['average_reaction_bad_answer_tot'] += prior_question_elapsed_time
cache[user_id]['part_score'][part]+=score
for tag in tags:
if tag in cache[user_id]['tag_count'].keys() :
cache[user_id]['tag_count'][tag] +=1
cache[user_id]['tag_good_answers'][tag] += answered_correctly
else:
cache[user_id]['tag_count'][tag] = 1
cache[user_id]['tag_good_answers'][tag] = answered_correctly
if(cache[user_id]['ts-1'] != -1)&(timestamp != cache[user_id]['ts-1']):
cache[user_id]['last_delta_between_questions'] =(timestamp - cache[user_id]['ts-1'])
 cache[user_id]['average_delta_between_questions'] +=(timestamp - cache[user_id]['ts-1'])
 cache = calculate_time_deltas(cache, timestamp, prior_question_elapsed_time,content_type_id, cache[user_id]['previous_part'], user_id, batch_size)
 cache[user_id]['ts-1'] = timestamp
cache[user_id]['last_ts_part'][part] = timestamp
cache[user_id]['previous_content_type_id'] = content_type_id
if part != -1:
cache[user_id]['previous_part'] = part
if content_type_id == 0:
cache[user_id]['previous_question'] = content_id
if content_type_id == 0:
cache[user_id]['last_questions'].append(( content_id,answered_correctly))
if len(cache[user_id]['last_questions'])>100:
cache[user_id]['last_questions'].pop(0)
 if content_type_id == 1:
part = lecture_to_part[content_id]
cache[user_id]['lectures_count']+=1
cache[user_id]['lectures_part'][part]+=1
cache[user_id]['last_lectures'].append(content_id)
 if len(cache[user_id]['last_lectures'])>100:
cache[user_id]['last_lectures'].pop(0)
 if(prior_question_had_explanation != -1)&(cache[user_id]['previous_question'] != -1):
cache[user_id]['question_with_explanation'].add(content_id)
 cache[user_id]['question_seen'].add(content_id)
 if cache[user_id]['first_bundle'] == -1:
cache[user_id]['first_bundle'] = content_id
cache[user_id]['batch_size_list'] = batch_size
return cache
batch_size = 64
epochs = 10
lr_reduce = ReduceLROnPlateau(monitor='val_acc', factor=0.1, epsilon=0.0001, patience=1, verbose=1 )<train_model>
def create_feature(cache, arr, batch_size, question_cache):
features = []
row_id, timestamp, user_id, content_id, content_type_id, task_container_id, prior_question_elapsed_time, prior_question_had_explanation, _, _ = arr
if not prior_question_elapsed_time:
prior_question_elapsed_time=-1
if not prior_question_had_explanation:
prior_question_had_explanation =-1
timestamp = float(timestamp)/1000
prior_question_elapsed_time = float(prior_question_elapsed_time)/1000
if content_type_id == 0:
part = questions_to_parts[content_id]
tags = questions_to_tag[content_id]
difficulty = questions_to_difficulty[content_id]
count = question_user_count[content_id]
bad_answer_reaction = bad_answer_reaction_time[content_id]
good_answer_reaction = good_answer_reaction_time[content_id]
var = questions_to_difficulty_variance[content_id]
if content_id in questions_to_difficulty_with_explanation.keys() :
difficulty1bis = questions_to_difficulty_with_explanation[content_id]
else:
difficulty1bis = -1
if content_id in questions_to_difficulty_without_explanation.keys() :
difficulty2bis = questions_to_difficulty_without_explanation[content_id]
else:
difficulty2bis = -1
kk = 0
features.append(part)
 features.append(difficulty)
 features.append(count)
 features.append(good_answer_reaction)
 features.append(bad_answer_reaction)
 features.append(batch_size)
 features.append(var)
 features.append(difficulty1bis)
 features.append(difficulty2bis)
 features.append(clust[content_id])
 features.append(cache[user_id]['lectures_count'])
 a=0
if user_id in cache.keys() :
features.append(cache[user_id]['student_parameters']['theta'])
 a+=1
else:
features.append(-1)
 if content_id in question_cache.keys() :
a+=1
else:
pass
if a ==2:
features.append(probability_of_good_answer(cache[user_id]['student_parameters']['theta'], question_cache[content_id]['beta'], 1/4))
else:
features.append(-1)
 if part in cache[user_id]['part_student_parameters'].keys() :
features.append(probability_of_good_answer(cache[user_id]['part_student_parameters'][part]['theta'], question_cache[content_id]['beta'], 1/4))
else:
features.append(-1)
 c = 0
p = 0
s = len(tags)
 for tag in tags:
c+=cache[user_id]['tag_student_parameters'][tag]['theta']
p+=probability_of_good_answer(cache[user_id]['tag_student_parameters'][tag]['theta'], question_cache[content_id]['beta'], 1/4)
 features.append(c/s)
 features.append(p/s)
 features.append(cache[user_id]['part_score'][part])
 features.append(( timestamp - cache[user_id]['ts-1'])/batch_size)
 features.append(list_feature_average(19, cache[user_id]['average_reactivities']))
if part in cache[user_id]['part_average_reactivity_tot'].keys() :
features.append(make_feature_average(cache[user_id]['part_count'][part], cache[user_id]['part_average_reactivity_tot'][part]))
else:
features.append(-1)
 features.append(make_feature_average(cache[user_id]['general_good_answer'], cache[user_id]['average_reaction_good_answer_tot']))
features.append(cache[user_id]['last_delta_between_questions'])
 features.append(list_feature_average(10,(cache[user_id]['time_between_questions'])))
 features.append(list_feature_average(5,(cache[user_id]['time_between_questions'])))
 if len(cache[user_id]['time_between_questions'])>1:
features.append(cache[user_id]['time_between_questions'][-1])
 else:
features.append(-1)
 features.append(make_feature_average(cache[user_id]['questions_count'], cache[user_id]['general_good_answer']))
features.append(list_feature_average(2, cache[user_id]['last_answers_correctly']))
features.append(list_feature_average(6, cache[user_id]['last_answers_correctly']))
features.append(list_feature_average(6, cache[user_id]['last_answers_correctly_part'][part]))
if content_id in cache[user_id]['question_with_explanation']:
features.append(1)
 else:
features.append(0)
 c = 0
c1 = 0
c0 = 0
p=0
if content_id in all_prob.keys() :
to_check = set(cache[user_id]['last_questions'] ).intersection(set(all_prob[content_id].keys()))
for content, answer in to_check:
p+=all_prob[content_id][(content,answer)]
if answer == 1:
c1+=1
else:
c0+=1
features.append(c1)
 if c1+c0:
features.append(p/(c1+c0))
else:
features.append(0)
 c = 0
p=0
if content_id in proba_lec.keys() :
to_check = set(cache[user_id]['last_lectures'] ).intersection(set(proba_lec[content_id].keys()))
for content in to_check:
p+=proba_lec[content_id][content]
c+=1
if c:
features.append(p/c)
 else:
features.append(0)
 return features
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[lr_reduce],
validation_data=(X_test, y_test),
epochs = epochs, verbose = 2 )<compute_test_metric>
def update_cache_batch(cache, df_arr, question_cache):
batch_array = []
task_init = -1
user_id0 = -1
for arr in df_arr:
user_id = arr[2]
timestamp = arr[1]
task_container_id = arr[5]
if user_id not in cache.keys() :
cache = create_cache(cache, user_id, timestamp, ids)
 if(task_container_id == task_init)&(user_id == user_id0):
batch_array.append(arr)
 else:
batch_size = len(batch_array)
 for arr_ in batch_array:
cache = update_cache(cache, arr_, batch_size, question_cache)
 task_init = task_container_id
if user_id != user_id0:
user_id0 = user_id
batch_array = []
batch_array.append(arr)
 batch_size = len(batch_array)
 for arr_ in batch_array:
cache = update_cache(cache, arr_, batch_size, question_cache)
 return cache
score = model.evaluate(X_test, y_test, verbose=0)
 print('valid loss:', score[0])
 print('valid accuracy:', score[1] )<predict_on_test>
def calculate_features(cache, df_arr, question_cache):
X = []
batch_array = []
task_init = -1
user_id0 = -1
for arr in df_arr:
user_id = arr[2]
timestamp = arr[1]
task_container_id = arr[5]
if user_id not in cache.keys() :
cache = create_cache(cache, user_id, timestamp, ids)
 if(task_container_id == task_init)&(user_id == user_id0):
batch_array.append(arr)
 else:
batch_size = len(batch_array)
 for arr_ in batch_array:
feats = create_feature(cache, arr_, batch_size, question_cache)
 if len(feats)>0:
X.append(feats)
 task_init = task_container_id
if user_id != user_id0:
user_id0 = user_id
batch_array = []
batch_array.append(arr)
 batch_size = len(batch_array)
 for arr_ in batch_array:
feats = create_feature(cache, arr_, batch_size, question_cache)
 if len(feats)>0:
X.append(feats)
 X = np.array(X)
 return X
pred = model.predict(test )<define_variables>
env = riiideducation.make_env()
iter_test = env.iter_test()
<create_dataframe><EOS> 
p_test_df = pd.DataFrame()
cache = {}
for idx,(test_df, _)in enumerate(iter_test):
test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(float)
 test_df = test_df.fillna(-1)
 submit_df = test_df.loc[test_df['content_type_id'] == False, ['row_id']].copy()
if not p_test_df.empty:
answered_correctly = [int(elmt)for elmt in test_df.iloc[0]['prior_group_answers_correct'][1:-1].replace(' ','' ).split(',')]
prior_group_responses = [int(elmt)for elmt in test_df.iloc[0]['prior_group_responses'][1:-1].replace(' ','' ).split(',')]
p_test_df['answered_correctly'] = answered_correctly
p_test_df['user_response'] = answered_correctly
test_arr = p_test_df.values
update_cache_batch(cache, test_arr, question_cache)
 X = calculate_features(cache, test_df.values, question_cache)
 if len(X):
preds = 1-lgb.predict_proba(X)[:,0]
else:
preds = None
submit_df['answered_correctly'] = preds
env.predict(submit_df)
 p_test_df = test_df