Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<data_type_conversions>
%matplotlib inline
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()<categorify>
im = Image.open("/kaggle/input/test-image/shutterstock_aero.jpg" )
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<categorify>
emboss = [-2,-1,0,-1,1,1,0,1,2]
kernel = ImageFilter.Kernel(( 3,3), emboss)
 im2 = im.filter(kernel)
 plt.figure(figsize=(10,8))
plt.xticks=[]
imshow(im2);
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()<feature_engineering>
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import Dropout
import numpy as np
import pandas as pd
from keras.utils import to_categorical
import collections
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<choose_model_class>
pylab.rc('figure', figsize=(10,7))
SMALL_SIZE = 8
MEDIUM_SIZE = 10
BIGGER_SIZE = 12
plt.rc('font', size=SMALL_SIZE)
 plt.rc('axes', titlesize=MEDIUM_SIZE)
 plt.rc('axes', labelsize=MEDIUM_SIZE)
 plt.rc('xtick', labelsize=MEDIUM_SIZE)
 plt.rc('ytick', labelsize=MEDIUM_SIZE)
 plt.rc('legend', fontsize=BIGGER_SIZE)
 plt.rc('figure', titlesize=BIGGER_SIZE )
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = len(skills)
 self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 return x, questions
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SAKT_model = SAKTModel(n_skill, embed_dim=128)
 try:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt"))
except:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt", map_location='cpu'))
SAKT_model.to(device)
 SAKT_model.eval()
group = pickle.load(open("group.pkl", "rb"))
print(psutil.virtual_memory().percent)
<split>
Train_data=pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 Test_data=pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
iter_test = env.iter_test()
prior_test_df = None<merge>
y=Train_data['label'].values
Train=Train_data.drop('label',axis=1 ).values
Test=Test_data.values
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 if(prior_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 parts = prior_test_df['part'].values
user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly, part in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets, parts):
user_sum_part_dict[user_id,part] += answered_correctly
user_count_part_dict[user_id,part]+= 1
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
user_sum_fail_dict[user_id]+=(1-answered_correctly)*(content_sum_dict[content_id]/content_count_dict[content_id])
 user_sum_std_fail_dict[user_id]+=(1-answered_correctly)*(( user_sum_fail_dict[user_id]/(user_count_dict[user_id])) -(content_sum_dict[content_id]/content_count_dict[content_id])) **2
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_sum_part = np.zeros(len(test_df), dtype=np.float32)
 user_count_part = np.zeros(len(test_df), dtype=np.int32)
 user_sum_fail=np.zeros(len(test_df), dtype=np.float32)
 user_sum_std_fail = np.zeros(len(test_df), dtype=np.float32)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id,part)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values,test_df['part'].values)) :
user_sum_part[i] = user_sum_part_dict[user_id,part]
user_count_part[i] = user_count_part_dict[user_id,part]
user_sum_std_fail[i] = user_sum_std_fail_dict[user_id]
user_sum_fail[i] = user_sum_fail_dict[user_id]
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df['mean_part_user'] = user_sum_part/(user_count_part)
 test_df['mean_fail_user'] = user_sum_fail/user_count
test_df[ 'deviation_fail_user']=user_sum_std_fail/user_count
test_df["count_user"]=user_count
test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
lgbm_final = sub_preds / len(clfs)
 test_df[target] = np.array(SAKT_outs)* 0.5 + lgbm_final * 0.5
env.predict(test_df[['row_id', target]] )<set_options>
def sort_dic(dic):
index=sorted(dic)
 sort_list=[]
label=[]
for i in index:
sort_list.append(dic[i])
 label.append(i)
 return sort_list,label
plt.style.use('seaborn-white')
 sns.set_style("white")
<load_from_disk>
train_count=collections.Counter(y)
 train_count,labels=sort_dic(train_count )
!unzip -o.. /input/tgs-salt-identification-challenge/train.zip -d./train/<load_from_disk>
X=Train.reshape(Train.shape[0],28,28,1)
 X_test=Test.reshape(Test.shape[0],28,28,1 )
!unzip -o.. /input/tgs-salt-identification-challenge/test.zip -d./test/<train_model>
X_t, X_v, y_t, y_v = train_test_split(X, y, test_size=0.1, random_state=42 )
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )<load_from_csv>
y_train=to_categorical(y_t,10)
 y_val=to_categorical(y_v,10)
 X_train=X_t/255
X_val=X_v/255
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 val_ids = pd.read_csv('.. /input/pseudolabelgen/validation_ids.csv')
 train_df = train_df.drop(val_ids.iloc[:,0].values)
 no_salt_ids = pd.read_csv(".. /input/pseudolabelgen/results(1)/no_salt_ids.csv", index_col="id", usecols=[0])
 no_salt_ids = no_salt_ids.sample(2340)
 some_salt_ids = pd.read_csv(".. /input/pseudolabelgen/results(1)/some_salt_ids.csv", index_col="id")
 some_salt_ids = some_salt_ids.sample(3660)
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
classifier=Sequential()
classifier.add(Conv2D(64,(3,3),input_shape=(28,28,1),activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))
 classifier.add(Dropout(0.2))
classifier.add(Conv2D(64,(3,3),activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))
 classifier.add(Dropout(0.2))
classifier.add(Flatten())
 classifier.add(Dense(units=256,activation='relu'))
classifier.add(Dropout(0.5))
classifier.add(Dense(units=10,activation='softmax'))
train_df["images"] = [np.array(load_img("./train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
train_df["masks"] = [np.array(load_img("./train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<categorify>
classifier.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'] )
def rle_decode(rle_mask):

 if str(rle_mask)==str(np.nan):
return np.zeros(( 101,101))
s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101 )<feature_engineering>
T=25
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1, 
)
 datagen.fit(X_train)
 history = classifier.fit_generator(datagen.flow(X_train,y_train, batch_size=128),
epochs = T, validation_data =(X_val,y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // 128 
)
train_df_temp = pd.DataFrame()
train_df_temp['id'] = no_salt_ids.index
train_df_temp = train_df_temp.set_index('id')
 train_df_temp = train_df_temp.join(depths_df)
 train_df_temp["images"] = [np.array(load_img("./test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(no_salt_ids.index)]
train_df_temp["masks"] = [np.zeros(( img_size_ori,img_size_ori)) for idx in tqdm_notebook(no_salt_ids.index)]
train_df = train_df.append(train_df_temp)
 del train_df_temp<feature_engineering>
y_pred=classifier.predict(X_val)
 pred_label=np.argmax(y_pred,axis=1)
 actual_label=np.argmax(y_val,axis=1 )
train_df_temp = pd.DataFrame()
train_df_temp['id'] = some_salt_ids.index
train_df_temp = train_df_temp.set_index('id')
 train_df_temp = train_df_temp.join(depths_df)
 train_df_temp["images"] = [np.array(load_img("./test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(some_salt_ids.index)]
train_df_temp["masks"] = [np.fliplr(np.rot90(rle_decode(some_salt_ids.loc[idx,'rle_mask']),-1)) for idx in tqdm_notebook(some_salt_ids.index)]
train_df = train_df.append(train_df_temp)
 del train_df_temp
gc.collect()<categorify>
error=pred_label!=y_v
missclassified_images=X_v[error]
missclassified_label=pred_label[error]
correct_label=y_v[error]
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<prepare_x_and_y>
pred_counts=collections.Counter(pred_label)
 labels_count=collections.Counter(y_v)
 prediction_count,labels=sort_dic(pred_counts)
 actual_count,labels=sort_dic(labels_count )
<prepare_x_and_y><EOS> 
submission=pd.read_csv("/kaggle/input/submission-mnist/submission.csv")
 submission.to_csv('submission.csv', index=False )