Kaggle competition: Digit Recognizer
TARGET = "answered_correctly"<categorify>
data_train = pd.read_csv(".. /input/train.csv")
 data_train.head()
QUESTION_FEATURES = ["part", "question_id", "lsi_topic"]
question_file = ".. /input/question-features-0102/questions.pickle"
questions_df = pd.read_pickle(question_file)[QUESTION_FEATURES]
questions_df["lsi_topic"] = questions_df["lsi_topic"].fillna(-1)
 questions_df["lsi_topic"] = questions_df["lsi_topic"].map(dict(map(lambda x:(x[1],x[0]), enumerate(questions_df["lsi_topic"].fillna(-1 ).unique()))))
questions_df["lsi_topic"] = questions_df["lsi_topic"] + 1<set_options>
y_train = data_train['label']
x_train = data_train.drop(labels = ["label"], axis = 1)
 print('Shape of whole data for training', data_train.shape)
 print('x_train:', x_train.shape)
 print('y_train:', y_train.shape)
 %matplotlib inline
def convert_to_grid(x_input):
N, H, W = x_input.shape
grid_size = int(ceil(sqrt(N)))
 grid_height = H * grid_size + 1 *(grid_size - 1)
 grid_width = W * grid_size + 1 *(grid_size - 1)
 grid = np.zeros(( grid_height, grid_width)) + 255
next_idx = 0
y0, y1 = 0, H
for y in range(grid_size):
x0, x1 = 0, W
for x in range(grid_size):
if next_idx < N:
img = x_input[next_idx]
low, high = np.min(img), np.max(img)
 grid[y0:y1, x0:x1] = 255.0 *(img - low)/(high - low)
 next_idx += 1
x0 += W + 1
x1 += W + 1
y0 += H + 1
y1 += H + 1
return grid
examples = np.array(x_train.iloc[:81] ).reshape(81, 28, 28)
 print(examples.shape)
 fig = plt.figure()
grid = convert_to_grid(examples)
 plt.imshow(grid.astype('uint8'), cmap='gray')
 plt.axis('off')
 plt.gcf().set_size_inches(7, 7)
 plt.title('Some examples of training data', fontsize=24)
 plt.show()
plt.close()
fig.savefig('training_examples.png')
 plt.close()
warnings.filterwarnings(action="ignore" )<load_pretrained>
x_test = pd.read_csv(".. /input/test.csv")
 x_test.head()
last_row_file = ".. /input/saint-final/last_row_states.pickle"
with open(last_row_file, "rb")as f:
last_row_states = pickle.load(f)
 def inference(iter_test, TARGET, saint_model, questions_df):
previous_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 previous_test_df = previous_test_df[previous_test_df.content_type_id == False]
previous_test_df["user_to_remove"] = False
previous_test_df["prior_question_elapsed_time"] =(previous_test_df["prior_question_elapsed_time"] // 100 ).clip(0,300)
 previous_test_df['current_container_size'] = previous_test_df[['user_id', 'task_container_id']].groupby(['user_id', 'task_container_id'])['task_container_id'].transform('size')
 if TEST_MODE: previous_test_df.to_csv("./previous_test_df1.csv")
 common_users = set(previous_test_df["user_id"].unique() ).intersection(set(last_row_states.keys()))
last_records = pd.DataFrame([
{**last_row_states[user_id], **{'user_id': user_id}} for user_id in common_users
])
 if len(last_records)!= 0:
previous_test_df = pd.concat([last_records, previous_test_df] ).reset_index(drop=True)
 previous_test_df = previous_test_df.sort_values(['user_id','timestamp'], ascending=True ).reset_index(drop = True)
 previous_test_df['last_timestamp'] = previous_test_df[['user_id', 'timestamp']].groupby(['user_id'])['timestamp'].shift(1, fill_value=0)
 previous_test_df['last_timestamp'] = previous_test_df[['user_id', 'task_container_id', 'last_timestamp']].groupby(['user_id', 'task_container_id'])['last_timestamp'].transform('first')
 previous_test_df['last_task_container_size'] = previous_test_df[['user_id', 'current_container_size']].groupby(['user_id'])['current_container_size'].shift(1, fill_value=0)
 previous_test_df['last_task_container_size'] = previous_test_df[['user_id', 'task_container_id', 'last_task_container_size']].groupby(['user_id', 'task_container_id'])['last_task_container_size'].transform('first')
 if TEST_MODE: previous_test_df.to_csv("./previous_test_df2.csv")
 previous_test_df['lag'] = previous_test_df['timestamp'] - previous_test_df['last_timestamp'] -(previous_test_df['prior_question_elapsed_time'] * previous_test_df['last_task_container_size'])
 previous_test_df["lag"] =(previous_test_df["lag"]//(100*60)).clip(0, 1440)
 if TEST_MODE: previous_test_df.to_csv("./previous_test_df3.csv")
 if TEST_MODE: previous_test_df.to_csv("./previous_test_df4.csv")
 previous_test_df = previous_test_df[previous_test_df["user_to_remove"] != True]
if TEST_MODE: previous_test_df.to_csv("./previous_test_df5.csv")
 prev_group = previous_test_df[[
'user_id',
'content_id',
'answered_correctly',
'lag',
'prior_question_elapsed_time',
'part',
'lsi_topic'
]].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values,
r['lag'].values,
r['prior_question_elapsed_time'].values,
r['part'].values,
r['lsi_topic'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:],
np.append(group[prev_user_id][2], prev_group[prev_user_id][2])[-MAX_SEQ:],
np.append(group[prev_user_id][3], prev_group[prev_user_id][3])[-MAX_SEQ:],
np.append(group[prev_user_id][4], prev_group[prev_user_id][4])[-MAX_SEQ:],
np.append(group[prev_user_id][5], prev_group[prev_user_id][5])[-MAX_SEQ:], 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1],
prev_group[prev_user_id][2],
prev_group[prev_user_id][3],
prev_group[prev_user_id][4],
prev_group[prev_user_id][5], 
)
 users_to_cache = previous_test_df.groupby("user_id" ).last()
user_ids = users_to_cache.index
timestamps = users_to_cache["timestamp"].values
content_ids = users_to_cache["content_id"].values
content_type_ids = users_to_cache["content_type_id"].values
task_container_ids = users_to_cache["task_container_id"].values
prior_question_elapsed_times = users_to_cache["prior_question_elapsed_time"].values
prior_question_had_explanations = users_to_cache["prior_question_had_explanation"].values
current_container_sizes = previous_test_df['current_container_size']
for row in zip(
 user_ids,
timestamps,
content_ids,
content_type_ids,
task_container_ids,
prior_question_elapsed_times,
prior_question_had_explanations,
current_container_sizes, 
):
user_id = row[0]
timestamp = row[1]
content_id = row[2]
content_type_id = row[3]
task_container_id = row[4]
prior_question_elapsed_time = row[5]
prior_question_had_explanation = row[6]
current_container_size = row[7]
row = {
"user_id": user_id,
"timestamp": timestamp,
"content_id": content_id,
"content_type_id": content_type_id,
"task_container_id": task_container_id,
"prior_question_elapsed_time": prior_question_elapsed_time,
"prior_question_had_explanation": prior_question_had_explanation,
"current_container_size": current_container_size,
}
row["user_to_remove"] = True
last_row_states[row["user_id"]] = row
test_df = pd.merge(test_df, questions_df[QUESTION_FEATURES], left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'].fillna(0, inplace = True)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df[TARGET] = 0
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
pt_x = item[2].to(device ).long()
lsi_x = item[3].to(device ).long()
c = item[4].to(device ).long()
et = item[5].to(device ).long()
lt = item[6].to(device ).long()
if TEST_MODE:
pd.DataFrame({
"x": x[0],
"target_id": target_id[0].detach().numpy() ,
"pt_x": pt_x[0].detach().numpy() ,
"lsi_x": lsi_x[0].detach().numpy() ,
"c": c[0].detach().numpy() ,
"et": et[0].detach().numpy() ,
"lt": lt[0].detach().numpy() ,
} ).to_csv("input.csv")
 with torch.no_grad() :
output = saint_model(x, target_id, pt_x, lsi_x, c, et, lt)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 pred = np.array(outs)
 test_df[TARGET] = pred
if not TEST_MODE and not MOCK_MODE:
set_predict(test_df[['row_id', TARGET]])
 print('Job Done')
 inference(iter_test, TARGET, saint_model, questions_df )<define_variables>
def pre_process_mnist(x_train, y_train, x_test):
x_train = x_train / 255.0
x_test = x_test / 255.0
batch_mask = list(range(41000, 42000))
x_validation = x_train[batch_mask]
y_validation = y_train[batch_mask]
batch_mask = list(range(41000))
x_train = x_train[batch_mask]
y_train = y_train[batch_mask]
mean_image = np.mean(x_train, axis=0)
 std = np.std(x_train, axis=0)
 for j in range(28):
for i in range(28):
if std[i, j, 0] == 0:
std[i, j, 0] = 1.0
x_train -= mean_image
x_validation -= mean_image
x_test -= mean_image
x_train /= std
x_validation /= std
x_test /= std
y_train = to_categorical(y_train, num_classes=10)
 y_validation = to_categorical(y_validation, num_classes=10)
 d_processed = {'x_train': x_train, 'y_train': y_train,
'x_validation': x_validation, 'y_validation': y_validation,
'x_test': x_test}
return d_processed
data = pre_process_mnist(x_train, y_train, x_test)
 for i, j in data.items() :
print(i + ':', j.shape)
global_test = 2 
<import_modules>
model = Sequential()
model.add(Conv2D(64, kernel_size=7, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=9, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=7, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=7, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(256, kernel_size=3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size=5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size=3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
import numpy as np
import pandas as pd
import glob
import riiideducation
import matplotlib.pyplot as plt
from tqdm import tqdm
from catboost import CatBoostClassifier
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler 
<set_options>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 **(x + epochs))
epochs = 50
h = model.fit(data['x_train'], data['y_train'], batch_size=100, epochs = epochs,
validation_data =(data['x_validation'], data['y_validation']), callbacks=[annealer], verbose=1)
pd.options.display.max_rows = 100
pd.options.display.max_columns = 100<data_type_conversions>
print("Epochs={0:d}, Train accuracy={1:.5f}, Validation accuracy={2:.5f}".format(epochs, max(h.history['acc']),
max(h.history['val_acc'])))
def df_to_np(df, filter_lectures:bool, convert_answers:bool):
tmstmp =(df['timestamp']/3600000 ).to_numpy(dtype = np.float32)
 userid = df['user_id'].to_numpy()
ctntid = df['content_id'].to_numpy()
ctnttp = df['content_type_id'].to_numpy()
contnr = df['task_container_id'].to_numpy()
pqtime = np.nan_to_num(df['prior_question_elapsed_time']\ 
.to_numpy(dtype = np.float32), nan = float32m1)
 pqexpl = df['prior_question_had_explanation']\ 
.to_numpy(dtype = np.int8, na_value = 1)
 if convert_answers:
usrans = df['user_answer'].to_numpy()
anscor = df['answered_correctly'].to_numpy()
if filter_lectures:
f = ctnttp == int8_0
if convert_answers:
return tmstmp[f], userid[f], ctntid[f], ctnttp[f],\
contnr[f], pqtime[f], pqexpl[f], usrans[f], anscor[f]
else:
return tmstmp[f], userid[f], ctntid[f], ctnttp[f],\
contnr[f], pqtime[f], pqexpl[f]
else:
return tmstmp, userid, ctntid, ctnttp, contnr, pqtime, pqexpl, usrans, anscor<load_from_csv>
model.save('my_model.h5' )
<load_from_csv><EOS> 
results = model.predict(data['x_test'])
 results = np.argmax(results, axis=1)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['Label'] = results
submission.to_csv('sample_submission.csv', index=None)