Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
warnings.simplefilter(action='ignore', category=FutureWarning)
 %matplotlib inline
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
np.set_printoptions(suppress=True, linewidth=np.nan, threshold=np.nan)
 train_file_csv = ".. /input/train.csv"
test_file_csv = ".. /input/test.csv"
x_train_file_npy = "x_train.npy"
y_train_file_npy = "y_train.npy"
x_test_file_npy = "x_test.npy"
x_train_cleaned_file_npy = "x_train_cleaned.npy"
y_train_cleaned_file_npy = "y_train_cleaned.npy"
model_path = ""
log_file = "log.txt"
log_mode = 'both'
def log_msg(msg=''):
fmt = "%H:%M:%S"
s = f'{datetime.today().strftime(fmt)}: {msg}'
if log_mode == 'file_only' or log_mode == 'both':
with open(log_file, "a")as myfile:
myfile.write(f'{s}
')
 if log_mode == 'screen_only' or log_mode == 'both':
print(s)
 log_msg('Script started.' )
add_a_cols = [c for c in train_lec.columns if c not in train.columns]
add_b_cols = [c for c in train.columns if c not in train_lec.columns]
for c in add_a_cols: train[c] = 0
for c in add_b_cols: train_lec[c] = 0
train_lec = train_lec[train.columns]
train_lec = reduce_mem_usage(train_lec)
 train = reduce_mem_usage(train)
 train = reduce_mem_usage(pd.concat([train, train_lec]))
del train_lec
gc.collect()
train.sort_values(by="row_id", inplace=True)
 train.reset_index(drop=True, inplace=True)
 train.shape<feature_engineering>
if not os.path.isfile(x_train_file_npy):
mnist_train_dataset = read_csv(train_file_csv, delimiter=',' ).values
y_train = mnist_train_dataset[:, 0]
y_train = y_train.astype('uint8')
 x_train = mnist_train_dataset[0:, 1:]
x_train = x_train.astype('uint8')
 np.save(x_train_file_npy, x_train)
 np.save(y_train_file_npy, y_train)
 if not os.path.isfile(x_test_file_npy):
mnist_test_dataset = read_csv(test_file_csv, delimiter=',' ).values
x_test = mnist_test_dataset
x_test = x_test.astype('uint8')
 np.save(x_test_file_npy, x_test)
sizes_all_user = np.array(train.user_id.value_counts().sort_index().values, dtype="int16")
 sizes_all_cumsum = sizes_all_user.cumsum()
t_ary = np.array(train[["user_id","timestamp"]])
 d_timestamp = get_user_items_dict(t_ary, sizes_all_cumsum)
 for key, item in d_timestamp.items() :
d_timestamp[key] = np.unique(item)
 del t_ary
gc.collect()<drop_column>
x_train = np.load(file=x_train_file_npy)
 y_train = np.load(file=y_train_file_npy)
 log_msg(f'x_train: {x_train.shape}.These are the training images.')
 log_msg(f'y_train: {y_train.shape}.These are the training labels.' )
train.drop(["bundle_id","question_rate_exp","user_answer"], axis=1, inplace=True)
 train.drop(train.loc[train.content_type_id==1].index, inplace=True)
 train.drop(["content_type_id"], axis=1, inplace=True)
 train.reset_index(drop=True, inplace=True)
 train["LorR"] = 0
train.loc[train.part.isin([5,6,7]), "LorR"] = 1
train = reduce_mem_usage(train )<predict_on_test>
lst = [12817,
60, 191, 2284, 2316, 5275, 7389, 19633, 19979, 24891, 29296, 32565, 38191, 38544, 40339, 41739,
4677, 7527, 9162, 13471, 16598, 20891, 27364,
240, 11593, 11896, 17966, 25708, 28560, 33198, 34477, 36018, 41492,
1383, 6781, 22478, 23604, 26171, 26182, 26411, 18593, 34862, 36051, 36241, 36830, 37544,
456, 2867, 2872, 5695, 6697, 9195, 18319, 19364, 27034, 29253, 35620,
7610, 12388, 12560, 14659, 15219, 18283, 24122, 31649, 40214, 40358, 40653,
6295, 7396, 15284, 19880, 20089, 21423, 25233, 26366, 26932, 27422, 31741,
8566, 10920, 23489, 25069, 28003, 28851, 30352, 30362, 35396, 36984, 39990, 40675, 40868, 41229,
631, 4226, 9943, 14914, 15065, 17300, 18316, 19399, 20003, 20018, 23135, 23732, 29524, 33641, 40881, 41354
]
x_cleaned = np.delete(x_train, lst, 0)
 y_cleaned = np.delete(y_train, lst, 0)
 np.save(x_train_cleaned_file_npy, x_cleaned)
 np.save(y_train_cleaned_file_npy, y_cleaned )
ar_history = np.hstack([unique_user_ids.reshape(-1,1),
hist_ans_cumsum.reshape(-1,1),
hist_num_cumsum.reshape(-1,1),
hist_exp_cumsum.reshape(-1,1),
hist_devp_cumsum.reshape(-1,1),
hist_devm_cumsum.reshape(-1,1),
hist_ar_etmie_rate.reshape(-1,1),
hist_ar_part,
hist_ar_ability])
 df_history = pd.DataFrame(ar_history, columns=["user_id","num_ans","num_que","num_exp","num_devp","num_devm","num_etime"]+part_cols+abi_cols)
 for c in df_history.columns:
if c not in ["num_devp","num_devm","num_etime"]:
df_history[c] = df_history[c].astype(int)
 df_history = reduce_mem_usage(df_history)
 df_history.sort_values(by="user_id", inplace=True)
 df_history.reset_index(drop=True, inplace=True)
 del hist_ans_cumsum, hist_num_cumsum, hist_exp_cumsum, hist_devp_cumsum, hist_devm_cumsum, hist_ar_etmie_rate, hist_ar_part, hist_ar_ability
gc.collect()
print(df_history.shape)
 df_history.head()<categorify>
x_train = np.load(file=x_train_cleaned_file_npy)
 y_train = np.load(file=y_train_cleaned_file_npy)
 log_msg(f'There are {x_train.shape[0]} training images.' )
def get_attempt_que(u, r, q):
if u in d.keys() :
ua = d[u]
return np.count_nonzero(( ua[:,0] < r)*(ua[:,1] == q))
return 0
def get_attempt_curt_part(u, r, p):
if u in d.keys() :
ua = d[u]
return np.count_nonzero(ua[ua[:,0]<r][-5:,2] == p)
 return 0
def get_attempt_tags(u, r, t):
if u in d.keys() :
ua = d[u]
return np.count_nonzero(( ua[:,0] < r)*(ua[:,3] == t))
return 0
def get_attempt_curt_ans_rate(u, r):
if u in d.keys() :
ua = d[u]
return np.mean(ua[ua[:,0]<r][-10:, 4])
 return 0
def get_lagtime(u, t, l):
if u in d_timestamp.keys() :
ul = d_timestamp[u]
ul = ul[ul < t]
if ul.shape[0] >= 3:
return ul[l]
elif ul.shape[0] == 2 and l != -3:
return ul[l]
elif ul.shape[0] == 1 and l == -1:
return ul[l]
return 0
def get_noplay_days(u, t):
if u in d_timestamp.keys() :
ul = d_timestamp[u] /(1000*3600)
 ul = ul[ul <= t]
if 1 < ul.shape[0]:
di = np.diff(ul)
 return np.where(di<24,0,di ).sum()
return 0
def get_attempt_thedaybefore(u, t, d):
if u in d_timestamp.keys() :
ul = d_timestamp[u] /(1000*3600)
 return np.count_nonzero(( t-d <= ul)*(ul < t))
return 0
def cut_and_merge_with_class(df, df_class, t_col, c_col):
df = df.copy()
df["base_class"] = pd.cut(df[t_col], [-0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1], labels=False)
 return df.merge(df_class[["question_id","base_class",c_col]], on=["question_id","base_class"], sort=False )<set_options>
random_seed = np.random.seed(2)
 x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.1, random_state=random_seed,
stratify=y_train )
data = train[train.user_id.isin(ext_user)].copy()
data.reset_index(drop=True, inplace=True)
 q_ary = np.array(train[["user_id","row_id","question_id","part","tag1","answered_correctly"]])
 q_ary = q_ary[np.argsort(q_ary[:,1])]
del train
gc.collect()
train = data.copy()
sizes_train_user = np.array(train.user_id.value_counts().sort_index().values, dtype="int16")
 sizes_train_cumsum = sizes_train_user.cumsum()
del data
gc.collect()
train.shape<feature_engineering>
log_msg('')
 log_msg('Changing data type to float and normalizing values...')
 x_train = x_train.astype('float32')
 x_train /= 255
x_val = x_val.astype('float32')
 x_val /= 255
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
 x_val = x_val.reshape(x_val.shape[0], 28, 28, 1)
 log_msg(f'x_train shape is {x_train.shape} of data type {x_train.dtype}.These are our TRAINING images.')
 log_msg(f'x_val shape is {x_val.shape} of data type {x_val.dtype}.These are our VALIDATION images.')
 y_train = np_utils.to_categorical(y_train, 10)
 y_train = y_train.astype('float32')
 y_val = np_utils.to_categorical(y_val, 10)
 y_val = y_val.astype('float32')
 log_msg(f'y_train shape is {y_train.shape} of data type {y_train.dtype}.These are our TRAINING labels.')
 log_msg(f'y_val shape is {y_val.shape} of data type {y_val.dtype}.These are our VALIDATION labels.' )
%%time
def rand_time(max_time_stamp):
interval = MAX_TIME_STAMP - max_time_stamp
rand_time_stamp = random.randint(0,interval)
 return rand_time_stamp
max_timestamp_u = train[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
MAX_TIME_STAMP = max_timestamp_u.max_time_stamp.max()
max_timestamp_u['rand_time_stamp'] = max_timestamp_u.max_time_stamp.apply(lambda x: rand_time(x))
train = train.merge(max_timestamp_u, on='user_id')
 train['viretual_time_stamp'] = train.timestamp + train['rand_time_stamp']
train.sort_values(['viretual_time_stamp', 'row_id'], inplace=True)
 train.reset_index(drop=True, inplace=True)
 train_num = int(train.shape[0] * TRAIN_RATE)
 if TUNE:
tune_num = int(train.shape[0] * TUNE_RATE)
 test_num = train.shape[0] -(train_num + tune_num)
 else:
test_num = train.shape[0] - train_num
tune_num = 0
print(train_num, test_num, tune_num)
 train["cv_group"] = 9
sta = 0
for c in range(CV):
end = int(train_num/CV)*(c+1)if(c+1)!= CV else train_num
train.loc[sta:end, "cv_group"] = c
print("CV:%s -- Start with %07d and end with %07d.Num of user is %s and mean correct answer rate is %f" %(c, sta, end, train.loc[train.cv_group==c].user_id.nunique() , train.loc[train.cv_group==c].answered_correctly.mean()))
sta = end
if TUNE:
train.loc[train_num+1:train_num+tune_num, "cv_group"] = 5
train.loc[train_num+tune_num+1:, "cv_group"] = 4
else:
train.loc[train_num+1:train_num+test_num, "cv_group"] = 4
train.sort_values('row_id', inplace=True)
 train.reset_index(drop=True, inplace=True )<data_type_conversions>
do_training = True
if do_training:
log_msg('---- TRAINING BEGIN ----')
 format_metric_val = '{:1.10f}'
metric = 'val_acc'
if metric == 'val_acc':
best_metric_val_so_far = 0
snifftest_max_epoch = 0
snifftest_metric_val = 0
elif metric == 'val_loss':
best_metric_val_so_far = 100.0
snifftest_max_epoch = 0
snifftest_metric_val = 100.0
iter_best_fit = 0
max_duration_mins = 330
fmt = "%a %b %d %H:%M:%S"
if max_duration_mins == 0:
started_at = 0
finish_by = 0
else:
started_at = datetime.today()
finish_by = started_at + timedelta(minutes=max_duration_mins)
 log_msg(f'Started at {started_at.strftime(fmt)}, finish by {finish_by.strftime(fmt)}')
 x = 3
epochs=500
patience=30
batch_size=500
for xtr in range(0, x):
K.clear_session()
log_msg(f'Iteration {xtr} of {x - 1}')
 input = Input(shape=(28, 28, 1))
x1 = Conv2D(32,(5, 5), padding='same', kernel_initializer='he_normal' )(input)
 x1 = Activation('relu' )(x1)
 x1 = Conv2D(64,(3, 3), padding='same', kernel_initializer='he_normal' )(x1)
 x1 = Activation('relu' )(x1)
 x1 = MaxPooling2D(pool_size=(2, 2))(x1)
 x1 = Dropout(0.5 )(x1)
 x1 = Conv2D(64,(3, 3), padding='same', kernel_initializer='he_normal' )(x1)
 x1 = Activation('relu' )(x1)
 x1 = Conv2D(32,(5, 5), padding='same', kernel_initializer='he_normal' )(x1)
 x1 = Activation('relu' )(x1)
 x1 = Dropout(0.5 )(x1)
 x1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x1)
 x1 = Dropout(0.5 )(x1)
 x1 = Flatten()(x1)
 x1 = Dense(128, activation='relu', kernel_initializer='he_normal' )(x1)
 x1 = Dropout(0.5 )(x1)
 output = Dense(10, activation='softmax', kernel_initializer='he_normal' )(x1)
 model = Model(inputs=[input], outputs=[output])
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 results, log = kbf.find_best_fit(model=model, metric=metric, xtrain=x_train, ytrain=y_train, xval=x_val,
yval=y_val, validation_split=0, batch_size=batch_size, epochs=epochs,
patience=patience, snifftest_max_epoch=snifftest_max_epoch,
snifftest_metric_val=snifftest_metric_val,
show_progress=True, format_metric_val=format_metric_val,
save_best=True, save_path=model_path,
best_metric_val_so_far=best_metric_val_so_far,
logmsg_callback=log_msg, finish_by=finish_by)
 del model
is_best = False
if metric == 'val_acc':
is_best = results['best_metric_val'] > best_metric_val_so_far
elif metric == 'val_loss':
is_best = results['best_metric_val'] < best_metric_val_so_far
if is_best:
iter_best_fit = xtr
best_metric_val_so_far = results['best_metric_val']
sbest_metric_val_so_far = metric + '=' + format_metric_val.format(best_metric_val_so_far)
 sbest_epoch = results['best_epoch']
best_log=results['history']
best_epoch=results['best_epoch']
s = f'NEW BEST SO FAR: {sbest_metric_val_so_far} on epoch {sbest_epoch}
'
log_msg(s)
 if results['expired']:
break
log_msg(f'The best result is {sbest_metric_val_so_far}')
 log_msg('---- TRAINING END ----' )
devp_rate =(devp_cumsum / num_cumsum ).astype("float32")
 devm_rate =(devm_cumsum / num_cumsum ).astype("float32")
 train["ans_rate"] = shift_to_prior(ans_rate, sizes_train_cumsum ).astype("float32")
 train["devp_rate"] = shift_to_prior(devp_rate, sizes_train_cumsum ).astype("float32")
 train["devm_rate"] = shift_to_prior(devm_rate, sizes_train_cumsum ).astype("float32")
 train["num_que"] = shift_to_prior(num_cumsum, sizes_train_cumsum ).astype("int16")
 train["num_exp"] = exp_cumsum.astype("int16")
 train["study_rate"] =(train.num_exp / train.num_que ).replace([np.inf,-np.inf], np.nan ).fillna(0 ).astype("float32")
 train = reduce_mem_usage(train)
 del ans_rate, devp_rate, devm_rate, num_cumsum, exp_cumsum
gc.collect()<categorify>
x_test = np.load(file=x_test_file_npy)
 x_test = x_test.astype('float32')
 x_test /= 255
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
 y_test = None
with open('.//model.json', 'r')as f:
modelx = model_from_json(f.read())
 modelx.load_weights('.//model.hdf5')
ar_part_rate = ar_part[:,:7] / ar_part[:,7:]
np.nan_to_num(ar_part_rate, copy=False)
 ar_part_rate = ar_part_rate.astype("float32")
 for i in range(ar_part_rate.shape[1]):
ar_part_rate[:,i] = shift_to_prior(ar_part_rate[:,i], sizes_train_cumsum)
 for i in range(ar_part.shape[1]):
ar_part[:,i] = shift_to_prior(ar_part[:,i], sizes_train_cumsum)
 ar_part_rate.shape, ar_part.shape<categorify>
ar_ability_rate = ar_ability[:,:8] / ar_ability[:,8:]
np.nan_to_num(ar_ability_rate, copy=False)
 ar_ability_rate = ar_ability_rate.astype("float32")
 for i in range(ar_ability_rate.shape[1]):
ar_ability_rate[:,i] = shift_to_prior(ar_ability_rate[:,i], sizes_train_cumsum)
 ar_ability_rate.shape<merge>
log_msg('Predicting a single test image...')
 img = x_test[0]
plt.imshow(img.reshape([28, 28]), cmap=plt.get_cmap('gray_r'))
plt.show()
img = img.reshape(1, 28, 28, 1)
 prediction = modelx.predict([img])
 predicted_classes = np.argmax(prediction[0], axis=0)
 log_msg('Probabilities')
 for x in range(0,10):
s = f'Class {x}: {format_metric_val.format(prediction[0,x])}'
log_msg(s)
 log_msg(f'Predicted class: {predicted_classes}')
train = train.join(pd.DataFrame(ar_part_rate, columns=part_rate_cols))
train = train.join(pd.DataFrame(ar_part[:,7:], columns=num_part_cols))
train = train.join(pd.DataFrame(ar_ability_rate, columns=abi_rate_cols))
train = train.join(pd.DataFrame(ar_etmie_rate, columns=["etime_rate"]))
train.sort_values("row_id", inplace=True)
 train.reset_index(drop=True, inplace=True)
 del ar_part_rate, ar_part, ar_ability_rate
gc.collect()
train.shape<data_type_conversions>
log_msg('Predicting all test images...')
 test_labels = modelx.predict(x_test)
 predicted_classes = np.argmax(test_labels, axis=1)
 log_msg('Prediction complete.' )
<feature_engineering><EOS> 
submission = pd.DataFrame({
'ImageID': range(1, 28001),
'Label': predicted_classes
})
 submission.to_csv('submission.csv', index=False)
 log_msg('Saved predictions as submission.csv')
 log_msg('Script ended.' )