Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<init_hyperparams>
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from keras.preprocessing.image import ImageDataGenerator, load_img
from keras.layers import Conv2D, BatchNormalization, Activation, MaxPooling2D, GlobalAveragePooling2D, Dense, Flatten, Dropout
from keras.optimizers import RMSprop, Adam, SGD
from keras import regularizers
from keras.callbacks import CSVLogger, ModelCheckpoint, ReduceLROnPlateau
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 400,
'max_bin':500,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.2,
"boosting_type": "gbdt",
"metric": 'auc',
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df)) 
<drop_column>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train.head(5 )
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
y = train['label']
train.drop('label', axis=1, inplace=True )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect() 
<choose_model_class>
y = to_categorical(y, num_classes = 10)
 y[0]
MAX_SEQ = 100
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim=128)
 try:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval() 
<data_type_conversions>
train = train.values.reshape(train.shape[0], 28, 28, 1)
 test = test.values.reshape(test.shape[0], 28, 28, 1)
 print('Reshaped Train set: ',train.shape, " & Reshaped Test Set", test.shape )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int)) 
<data_type_conversions>
train = train.astype("float32")/255.0
test = test.astype("float32")/255.0
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
X_train, X_val, y_train, y_val = train_test_split(train, y, test_size=0.25, random_state=0)
 print("Number of samples in Training set :", X_train.shape[0])
 print("Number of samples in Validation set :", X_val.shape[0] )
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()<categorify>
train_datagen = ImageDataGenerator(rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 training_set = train_datagen.flow(X_train, y_train,
batch_size=64 
)
 val_datagen = ImageDataGenerator()
val_set = val_datagen.flow(X_val, y_val,
batch_size=64
)
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()<categorify>
model = tf.keras.models.Sequential()
model.add(Conv2D(64, kernel_size=(5,5), padding='same', activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(64, kernel_size=(5,5), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(128, kernel_size=(3,3), activation='relu', padding='same'))
model.add(Conv2D(128, kernel_size=(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<feature_engineering>
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=4,
verbose=1,
min_delta=0.0001 )
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<feature_engineering>
steps_per_epoch = training_set.n // training_set.batch_size
validation_steps = val_set.n // val_set.batch_size
hist = model.fit(x=training_set,
validation_data=val_set,
epochs=35,
callbacks=[reduce_lr],
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps
)
<define_variables>
_, acc_val = model.evaluate(val_set)
 _, acc_tr = model.evaluate(val_set)
 print("
Final Accuracy on training set : {:.2f}% & accuracy on validation is set: {:.2f}%".format(acc_tr*100, acc_val*100))
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None<define_search_space>
pred = model.predict(test)
 res = np.argmax(pred, axis=1)
 submission = pd.DataFrame({"ImageId":[i+1 for i in range(len(test)) ],
"Label": res})
 submission.head(10 )
N=[0.4,0.6]<feature_engineering>
submission.to_csv("submission.csv", index=False )
<import_modules><EOS> 
submission.to_csv("submission.csv", index=False )