Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<data_type_conversions>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras import layers
from keras import models
from keras import optimizers
from keras.utils import to_categorical
from sklearn.utils import shuffle
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from skimage import img_as_ubyte
from skimage.transform import resize
from scipy.ndimage.interpolation import rotate
def get_cor_table() :
max_neigbrs = 1000
global cor_table
map1 = np.load('.. /input/content-correlation-100to300/ctnt_map.npy')
 map2 = np.load('.. /input/content-correlation/ctnt_map.npy')
 cor1 = np.load('.. /input/content-correlation-100to300/result.npy')
 cor2 = np.load('.. /input/content-correlation/result.npy')
 cor =(cor2[map2[:max_content],:,:][:,map2[:max_content],:].astype(np.uint32)
 +cor1[map1[:max_content],:,:][:,map1[:max_content],:].astype(np.uint32))
size = np.log(np.log(cor[:,:,0].sum(axis = 1)+1))
size = size/size.max() /10
corrs =(cor[:,:,0]+5)/(cor[:,:,1]*1.7+cor[:,:,0]+10)+size
cor_table = corrs.argsort(axis = 1)[:,-max_neigbrs:].astype(np.int32)
<create_dataframe>
train_data = pd.read_csv('/kaggle/input/train.csv')
 test_data = pd.read_csv('/kaggle/input/test.csv' )
def get_content_answer_shares() :
global ca_shares_all
columns = ['user_id',
'content_id',
'content_type_id',
'user_answer',
'answered_correctly']
df = get_train_large(t_part=99, columns=columns)
 df = df.loc[df.content_type_id == 0, df.columns != 'content_type_id']
ca_shares_all = pd.pivot_table(df,
values='answered_correctly',
index='content_id',
columns='user_answer',
aggfunc='count',
fill_value=0 ).to_numpy() +1
ca_shares_all =(ca_shares_all.T/ca_shares_all.sum(axis = 1)).T.astype(np.float32 )<groupby>
X_model = train_data.drop('label', axis=1)
 y_model = train_data['label'].copy()
Y_finish = test_data
print('size train data:', X_model.shape)
 print('size train labels:', y_model.shape)
 print('size finish test data:', Y_finish.shape )
def get_content_first_answer_mean() :
global ctnt_fam
columns = ['user_id',
'new_order',
'answered_correctly',
'content_id',
'content_type_id']
df = get_train_large(t_part = 99, columns = columns)
 df = df\ 
.loc[df.content_type_id==0, df.columns!='content_type_id']\ 
.sort_values(by = 'new_order')
 df = df.groupby(['user_id','content_id'] ).first()
df = df.groupby('content_id' ).answered_correctly.mean().sort_index()
ctnt_fam = df.to_numpy(dtype = np.float32 )<normalization>
def img_rotate(df_x, angle):

 change_img = np.empty([df_x.shape[0], df_x.shape[1]])
 for i, image in enumerate(df_x.values):
img = rotate(image.reshape(28, 28), angle, cval=0, reshape=False, order=0)
 change_img[i] = img.ravel()
return pd.DataFrame(data=change_img, columns=df_x.columns)
 def img_zoom(df_x, scale):

 if(scale > 0.9)or(scale < -0.9):
raise Exception('scale values must be between -0.9 and 0.9')
 if scale < 0:
change_img = np.empty([df_x.shape[0], df_x.shape[1]])
 for i, image in enumerate(df_x.values):
number_around = round(28*abs(scale)/2)
 img = image.reshape(28, 28 ).astype('float32' ).copy()
img = np.pad(img, number_around, mode='constant')
 img = resize(img,(28,28), anti_aliasing=False, mode='constant', cval=0, order=0)
 change_img[i] = img.ravel().astype('int64')
 return pd.DataFrame(data=change_img, columns=df_x.columns)
 elif scale >= 0:
change_img = np.empty([df_x.shape[0], df_x.shape[1]])
 for i, image in enumerate(df_x.values):
number_slice = round(28*abs(scale)/2)
 img = image.reshape(28, 28 ).astype('float32' ).copy()
img = img[number_slice:28-number_slice, number_slice:28-number_slice]
img = resize(img,(28,28), anti_aliasing=False, mode='constant', cval=0, order=0)
 change_img[i] = img.ravel().astype('int64')
 return pd.DataFrame(data=change_img, columns=df_x.columns)
def get_ctnt_enc() :
global ctnt_enc
qestn_tagsmap_ohe = np.zeros(( len(qestn_tagsmap), 189), dtype = np.bool)
 for i,j in enumerate(qestn_tagsmap):
for k in j:
qestn_tagsmap_ohe[i,k] = True
tags_comps = StandardScaler().fit_transform(
 PCA(n_components=3, random_state=0 ).fit_transform(qestn_tagsmap_ohe)
)
 corr_comps = StandardScaler().fit_transform(
 PCA(n_components=9, random_state=0 ).fit_transform(cor_table)
)
 comb_comps = np.concatenate([tags_comps,corr_comps], axis = 1)
 ctnt_enc = PCA(n_components=1, random_state=0)\ 
.fit_transform(comb_comps ).astype(np.float32 ).ravel()<load_pretrained>
X_train, X_test, y_train, y_test = train_test_split(X_model, y_model, test_size=0.2 )
def get_train_small(t_part:int):
all_files = glob.glob('.. /input/riiid-parquets-v5/df_*')
 read_files = [file for file in all_files if file.endswith('_'+str(t_part)) ]
df = pd.read_parquet(read_files[0])
 return df<load_pretrained>
X_train_add = X_train.append(img_zoom(X_train, 0.2))
X_train_add = X_train_add.append(img_zoom(X_train, -0.3))
X_train_add = X_train_add.append(img_rotate(X_train, 11))
X_train_add = X_train_add.append(img_rotate(X_train, -11))
y_train_add = y_train.append(y_train)
 y_train_add = y_train_add.append(y_train)
 y_train_add = y_train_add.append(y_train)
 y_train_add = y_train_add.append(y_train)
 X_train = X_train_add.copy()
y_train = y_train_add.copy()
X_train, y_train = shuffle(X_train, y_train )
def get_train_large(t_part:int, columns:list):
all_files = glob.glob('.. /input/riiid-parquets-v5/df_*')
 read_files = [file for file in all_files if not file.endswith('_'+str(t_part)) ]
df = pd.concat([pd.read_parquet(file, columns = columns)for file in read_files])
 return df<groupby>
X_train = X_train.values.reshape(X_train.shape[0], 28, 28 ,1)
 X_train = X_train.astype('float32')/ 255
X_test = X_test.values.reshape(X_test.shape[0], 28, 28, 1)
 X_test = X_test.astype('float32')/ 255
Y_finish = Y_finish.values.reshape(Y_finish.shape[0], 28, 28 ,1)
 Y_finish = Y_finish.astype('float32')/ 255
def get_train_groups(t_part:int):
df = get_train_small(t_part)
 groups = []
for i in np.arange(0, 10000, dtype = np.int16):
group = df.loc[df.new_order == i].reset_index(drop = True)
 groups.append(group)
 return groups<data_type_conversions>
def build_model() :
model = models.Sequential()
model.add(layers.Conv2D(32,(3,3), activation='relu', input_shape=(28,28,1)))
 model.add(layers.MaxPooling2D(( 2,2)))
 model.add(layers.Dropout(0.12))
model.add(layers.Conv2D(64,(3,3), activation='relu'))
model.add(layers.MaxPooling2D(( 2,2)))
 model.add(layers.Dropout(0.12))
model.add(layers.Conv2D(64,(3,3), activation='relu'))
model.add(layers.Flatten())
 model.add(layers.Dropout(0.12))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
opt = optimizers.Adam(lr=0.0015, beta_1=0.9, beta_2=0.99, epsilon=None, decay=0.0, amsgrad=False)
 model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
 return model
def get_arrays_and_lists() :
global next_uplace,\
au_ctntid,\
a_userid,\
lu_seq,\
lu_seq_part,\
au_anshar,\
au_ctshar,\
user_map,\
au_tmstmp_prv
au_ctntid = np.zeros(( max_users, max_content, 3), dtype = np.int8)
 a_userid = np.zeros(( max_users, 2), dtype = np.int16)
 au_anshar = np.zeros(( max_users, 2), dtype = np.float32)
 au_ctshar = np.zeros(( max_users, 2), dtype = np.float32)
 user_map = np.zeros(np.iinfo(np.int32 ).max,dtype = np.int32)
 next_uplace = np.int32(1)
 au_tmstmp_prv = np.zeros(( max_users,3), dtype = np.float32)
 lu_seq = [[] for _ in range(max_users)]
lu_seq_part = [[[],[],[],[],[],[],[]] for _ in range(max_users)] 
<categorify>
y_train = to_categorical(y_train, 10)
 y_test = to_categorical(y_test, 10)
 cnn = build_model()
cnn.fit(X_train,
y_train,
epochs=4,
batch_size=64 )
def update_user_map(unique_users):
global next_uplace
for u in unique_users:
if user_map[u] == int32_0:
user_map[u] = next_uplace
next_uplace += int32_1<feature_engineering>
test_loss, test_acc = cnn.evaluate(X_test, y_test)
 test_acc
def update_arrays(df):
tmstmp,userid,ctntid,ctnttp,contnr,pqtime,pqexpl,usrans,anscor = df_to_np(df,False,True)
 for r in range(len(df)) :
user_ = user_map[userid[r]]
if tmstmp[r] > au_tmstmp_prv[user_,0]:
au_tmstmp_prv[user_,2] = au_tmstmp_prv[user_,1]
au_tmstmp_prv[user_,1] = au_tmstmp_prv[user_,0]
au_tmstmp_prv[user_,0] = tmstmp[r]
if ctnttp[r] == int8_0:
lsu = lu_seq[user_]
lsup = lu_seq_part[user_][qestn_partmap[ctntid[r]]]
bndl_ = qestn_bndlmap[ctntid[r]]
ctnt_ = ctntid[r]
if len(lsu)>m: lsu.pop(0)
 if len(lsup)>m: lsup.pop(0)
 au_ctntid[user_,ctnt_,1] += int8_1
au_ctntid[user_,ctnt_,2] = usrans[r]
if anscor[r] == int8_1:
a_userid[user_,0] += int16_1
au_ctntid[user_,ctnt_,0] = int8_1
lsu.append(True)
 lsup.append(True)
 au_anshar[user_, 0] += ca_shares_all[ctnt_,usrans[r]]
au_ctshar[user_, 0] += ctnt_fam[ctnt_]
else:
a_userid[user_,1] += int16_1
au_ctntid[user_,ctnt_,0] = int8_0
lsu.append(False)
 lsup.append(False)
 au_anshar[user_, 1] += ca_shares_all[ctnt_,usrans[r]]
au_ctshar[user_, 1] += ctnt_fam[ctnt_] 
<categorify>
predict_test = cnn.predict_classes(X_test)
 y_correct = np.argmax(y_test, axis=1)
 correct_idx = np.nonzero(predict_test==y_correct)
 incorrect_idx = np.nonzero(predict_test!=y_correct )
def get_features(df, is_test:bool):
if is_test:
tmstmp,userid,ctntid,ctnttp,contnr,pqtime,pqexpl=\
df_to_np(df,True,False)
 else:
tmstmp,userid,ctntid,ctnttp,contnr,pqtime,pqexpl,usrans,anscor=\
df_to_np(df,True,True)
 user = user_map[userid]
part = qestn_partmap[ctntid]
userid_ctntid_ = au_ctntid[user,ctntid]
userid_ = a_userid[user,:]
userid_avg_ =(userid_[:,0]/(userid_[:,0]+userid_[:,1]+int16_1)).astype(np.float32)
 cp_ = ac_pqexpl[ctntid,pqexpl,:]
ctntid_pqexpl_avg_ =(cp_[:,0]/(cp_[:,0]+cp_[:,1]+int32_1)).astype(np.float32)
 uanshar_ = au_anshar[user, :]
uanshar_fls =(uanshar_[:,1]/(userid_[:,1]+e)).astype(np.float32)
 uctshar_ = au_ctshar[user, :]
uctshar_slf =(uctshar_[:,0]/(uctshar_[:,0]+uctshar_[:,1]+e)).astype(np.float32)
 uctshar_cor =(uctshar_[:,0]/(userid_[:,0]+e)).astype(np.float32)
 uctshar_fls =(uctshar_[:,1]/(userid_[:,1]+e)).astype(np.float32)
 correlation_ids = cor_table[ctntid]
neigh = au_ctntid[user.reshape(-1,1),correlation_ids,:]
all_ans_cnt = np.count_nonzero(neigh[:,:,1],axis = 1 ).astype(np.int16)
 cor_ans_cnt = np.count_nonzero(neigh[:,:,0],axis = 1 ).astype(np.int16)
 fls_ans_cnt = all_ans_cnt - cor_ans_cnt
neigh_ca_shrs_all = ca_shares_all[correlation_ids,neigh[:,:,2]]*(neigh[:,:,1]!=int8_0)
 cor_shrs_all =(( neigh_ca_shrs_all*(neigh[:,:,0]==int8_1)).sum(axis = 1)/ 
(cor_ans_cnt+e)).astype(np.float32)
 fls_shrs_all =(( neigh_ca_shrs_all*(neigh[:,:,0]==int8_0)).sum(axis = 1)/ 
(fls_ans_cnt+e)).astype(np.float32)
 lu_seq_ = [lu_seq[u] for u in user]
lst_m_avg = np.array(
 [x.count(True)/(len(x)+e)for x in lu_seq_],
dtype = np.float32)
 lst_s_avg = np.array(
 [x[-s:].count(True)/(len(x[-s:])+e)for x in lu_seq_],
dtype = np.float32)
 lu_seq_part_ = [lu_seq_part[u][part[_]] for _, u in enumerate(user)]
lst_part_m_avg = np.array(
 [x.count(True)/(len(x)+e)for x in lu_seq_part_],
dtype = np.float32)
 lst_part_s_avg = np.array(
 [x[-s:].count(True)/(len(x[-s:])+e)for x in lu_seq_part_],
dtype = np.float32)
 X = pd.DataFrame({
'part':part,
'prior_explanation':pqexpl,
'prior_elapsed_time':pqtime,
'content':ctntid,
'ctntent_encoded':ctnt_enc[ctntid],
'task_container':contnr,
'time_to_cont_1':tmstmp - au_tmstmp_prv[user,0],
'time_to_cont_3':tmstmp - au_tmstmp_prv[user,2],
'time_cont1_to_cont2':au_tmstmp_prv[user,0] - au_tmstmp_prv[user,1],
'time_cont2_to_cont3':au_tmstmp_prv[user,1] - au_tmstmp_prv[user,2],
'user_content_attempts':userid_ctntid_[:,1],
'user_content_last_1':userid_ctntid_[:,0],
'user_part_last_m_avg':lst_part_m_avg,
'user_part_last_s_avg':lst_part_s_avg,
'user_last_m_avg':lst_m_avg,
'user_last_s_avg':lst_s_avg,
'content_explanation_avg':ctntid_pqexpl_avg_,
'content_first_answer_avg':ctnt_fam[ctntid],
'content_avg_time':ctnt_mtime[ctntid],
'user_relative_content_avg':uctshar_slf,
'user_true_content_avg':uctshar_cor,
'user_false_content_avg':uctshar_fls,
'user_false_answer_avg':uanshar_fls,
'neighbor_content_true_shares':cor_shrs_all,
'neighbor_content_false_shares':fls_shrs_all,
})
 if is_test:
return X
else:
return X, anscor 
<init_hyperparams>
target_names = ["Class {}".format(i)for i in range(10)]
print(classification_report(y_correct, predict_test, target_names=target_names))
%%time
uint8_0 = np.uint8(0)
 uint8_1 = np.uint8(1)
 uint16_0 = np.uint16(0)
 uint16_1 = np.uint16(1)
 int8_0 = np.int8(0)
 int8_1 = np.int8(1)
 int16_0 = np.int16(0)
 int16_1 = np.int16(1)
 int32_0 = np.int32(0)
 int32_1 = np.int32(1)
 float32m1 = np.float32(-1)
 max_users = 450000
max_content = 13523
m = 100
s = 20
e = 0.1
question_maps()
lecture_maps()
get_cor_table()
get_content_answer_shares()
get_ac_pqexpl()
get_content_first_answer_mean()
get_ctnt_enc()
ctnt_mtime = np.load('.. /input/question-duration/question_mean_time.npy' )<categorify>
predict = cnn.predict_classes(Y_finish )
%%time
if global_test == 1:
for i in tqdm(range(10)) :
X = []
y = []
get_arrays_and_lists()
groups = get_train_groups(i)
 for df in groups:
update_user_map(df.user_id.unique())
 X_, y_ = get_features(df,False)
 X.append(X_)
 y.append(y_)
 update_arrays(df)
 del(groups)
 X = pd.concat(X)
 y = np.concatenate(y)
 X.to_parquet('X_'+str(i))
np.save('y_'+str(i), y )<load_pretrained>
df_out = pd.DataFrame({'ImageId': range(1, len(predict)+1),
'Label': predict} )
<split><EOS> 
df_out.to_csv('mnist_cnn.csv', index=False, header=True )