Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
BATCH_SIZE = 64
VALID_BATCH_SIZE = 100
TEST_BATCH_SIZE = 100
EPOCHS = 5
NUM_CLASSES = 10
SEED = 42
EARLY_STOPPING = 25
OUTPUT_DIR = '/kaggle/working/'
MODEL_NAME = 'efficientnet-b0'
TARGET = 'answered_correctly'
FEATS_1 = ['mean_user_accuracy',
'answered_count',
'mean_content_accuracy_sm',
'prior_question_elapsed_time',
'last_incorrect_time', 'prior_question_wait_time',
'content_freq_encoding',
'lag_time',
'attempt_no', 'last_lecture_time',
'mean_user_spent_time_part',
'answered_correctly_sum_user_part',
'mean_user_accuracy_part',
'part', 'theta', 'beta',
'question_avg_explanation_sm',
'question_avg_elapsed_time_sm',
'tags_lsi',
'difficulty_incorrect', 'difficulty_diff'
]
categorical_features = ['part', 'tags_lsi']<define_variables>
!pip install efficientnet-pytorch
FEATS_2 = ['mean_user_accuracy',
'answered_correctly_sum_user',
'answered_count',
'mean_content_accuracy_sm',
'prior_question_elapsed_time',
'hmean_user_content_accuracy',
'last_incorrect_time', 'prior_question_wait_time',
'content_freq_encoding',
'lag_time',
'attempt_no', 'last_lecture_time',
'mean_user_spent_time_part',
'answered_correctly_sum_user_part',
'mean_user_accuracy_part',
'part', 'theta', 'beta',
'question_avg_explanation_sm',
'question_avg_elapsed_time_sm',
'tags_lsi',
'difficulty_incorrect',
]<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.autograd import Variable
from torch.utils.data import DataLoader, Dataset
from sklearn.metrics import accuracy_score
from PIL import Image, ImageOps, ImageEnhance
from efficientnet_pytorch import EfficientNet
model_1 = lgb.Booster(model_file='.. /input/lgbm-inference-db-full-data/lightgbm_v11.5.txt' )<define_variables>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print('Shape of the training data: ', train.shape)
 print('Shape of the test data: ', test.shape )
model_2 = lgb.Booster(model_file='.. /input/lgbm-inference-db-full-data/lightgbm_v11.6.txt' )<predict_on_test>
train_df, valid_df = train_test_split(train, test_size = 0.2, random_state=SEED,stratify=train['label'] )
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<concatenate>
n_pixels = len(train_df.columns)- 1
class MNIST_Dataset(Dataset):

 def __init__(self, df 
):
if len(df.columns)== n_pixels:
self.X = df.values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = None
self.X3 = np.full(( self.X.shape[0], 3, 28, 28), 0.0)
 for i, s in enumerate(self.X):
self.X3[i] = np.moveaxis(cv2.cvtColor(s, cv2.COLOR_GRAY2RGB), -1, 0)
 else:
self.X = df.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = torch.from_numpy(df.iloc[:,0].values)
 self.X3 = np.full(( self.X.shape[0], 3, 28, 28), 0.0)
 for i, s in enumerate(self.X):
self.X3[i] = np.moveaxis(cv2.cvtColor(s, cv2.COLOR_GRAY2RGB), -1, 0)
 def __len__(self):
return len(self.X3)
 def __getitem__(self, idx):
if self.y is not None:
return self.X3[idx] , self.y[idx]
else:
return self.X3[idx]
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
test_df = pd.concat([test_df.reset_index(drop=True),
questions_df.reindex(test_df['content_id'].values ).reset_index(drop=True)], axis=1)
 test_df = pd.concat([test_df.reset_index(drop=True),
part_df.reindex(test_df['part'].values ).reset_index(drop=True)], axis=1)
 if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df)
 else:
bundle_count = 1
temp_values = np.empty(10)* np.nan
test_df, bundle_count, temp_values = calc_user_feats_test(test_df, bundle_count, temp_values)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['part'] = test_df.part.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 for col in categorical_features:
test_df[col] = test_df[col].astype('category')
 test_df[TARGET] = model_1.predict(test_df[FEATS_1])* 0.5 + model_2.predict(test_df[FEATS_2])* 0.5
set_predict(test_df[['row_id', TARGET]] )<split>
train_dataset = MNIST_Dataset(train_df)
 valid_dataset = MNIST_Dataset(valid_df)
 test_dataset = MNIST_Dataset(test)
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
 valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset,
batch_size=VALID_BATCH_SIZE, shuffle=False)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=TEST_BATCH_SIZE, shuffle=False )
env = riiideducation.make_env()
iter_test = env.iter_test()<import_modules>
def get_model(model_name='efficientnet-b0'):
model = EfficientNet.from_pretrained(model_name)
 del model._fc
model._fc = nn.Linear(1280, NUM_CLASSES)
 return model
import sys
import numpy as np<set_options>
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed )
warnings.filterwarnings("ignore")
<set_options>
set_seed(SEED)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 output_dir = OUTPUT_DIR
model = get_model(MODEL_NAME)
 model = model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=0.001)
 scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
 loss_func = nn.CrossEntropyLoss()
if torch.cuda.is_available() :
model = model.cuda()
loss_func = loss_func.cuda()
best_val_accuracy = 0
min_val_loss = np.inf
best_epoch = 0
batches = 0
epochs_no_improve = 0
n_epochs_stop = EARLY_STOPPING
for epoch in range(EPOCHS):
running_loss = 0.0
targets = torch.empty(size=(BATCH_SIZE,)).to(device)
 outputs = torch.empty(size=(BATCH_SIZE,)).to(device)
 model.train()
for batch_idx,(data, target)in enumerate(train_loader):
batches += 1
data, target = Variable(data), Variable(target)
 if torch.cuda.is_available() :
data = data.type(torch.FloatTensor ).cuda()
target = target.cuda()
targets = torch.cat(( targets, target), 0)
 optimizer.zero_grad()
output = model(data)
 loss = loss_func(output, target)
 output = torch.argmax(torch.softmax(output, dim=1), dim=1)
 outputs = torch.cat(( outputs, output), 0)
 running_loss += loss.item()
loss.backward()
optimizer.step()
scheduler.step()
print('train/loss on EPOCH {}: {}'.format(epoch, running_loss/batches))
train_acc = accuracy_score(targets.cpu().detach().numpy().astype(int),
outputs.cpu().detach().numpy().astype(int))
print('train/accuracy: {} for epoch {}'.format(train_acc, epoch))
model.eval()
running_loss = 0.0
batches = 0
targets = torch.empty(size=(BATCH_SIZE,)).to(device)
 outputs = torch.empty(size=(BATCH_SIZE,)).to(device)
 for batch_idx,(data, target)in enumerate(valid_loader):
batches += 1
data, target = Variable(data, volatile=True), Variable(target)
 if torch.cuda.is_available() :
data = data.type(torch.FloatTensor ).cuda()
target = target.cuda()
with torch.no_grad() :
targets = torch.cat(( targets, target), 0)
 output = model(data)
 loss = loss_func(output, target)
 output = torch.argmax(torch.softmax(output, dim=1), dim=1)
 outputs = torch.cat(( outputs, output), 0)
 running_loss += loss.item()
val_loss = running_loss/batches
print('val/loss: {}'.format(val_loss))
val_acc = accuracy_score(targets.cpu().detach().numpy().astype(int),
outputs.cpu().detach().numpy().astype(int))
print('val/accuracy: {} for epoch {}'.format(val_acc, epoch))
if val_acc > best_val_accuracy:
best_val_accuracy = val_acc
min_val_loss = val_loss
print('Best val/acc: {} for epoch {}, saving model---->'.format(val_acc, epoch))
torch.save(model.state_dict() , "{}/snapshot_epoch_{}.pth".format(output_dir, epoch))
best_epoch = epoch
epochs_no_improve = 0
else:
epochs_no_improve += 1
if epochs_no_improve == n_epochs_stop:
print('Early stopping!')
 break
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 q_pad = 13523
a_pad = 3
start_token = 2<load_pretrained>
def prediction(model, data_loader):
model.eval()
test_pred = torch.LongTensor()
for i, data in enumerate(data_loader):
data = Variable(data, volatile=True)
 if torch.cuda.is_available() :
data = data.type(torch.FloatTensor ).cuda()
output = model(data)
 pred = output.cpu().data.max(1, keepdim=True)[1]
test_pred = torch.cat(( test_pred, pred), dim=0)
 return test_pred
group = pd.read_pickle(".. /input/groups/group.pandas" )<categorify>
model.load_state_dict(torch.load("snapshot_epoch_{}.pth".format(best_epoch)))
 test_pred = prediction(model, test_loader)
 submission = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy() ],
columns=['ImageId', 'Label'])
features_1_path = '.. /input/get-features-1/'
que_data = pd.read_pickle(features_1_path + "que_data.pickle")
 difficulty =(np.round(que_data.que_correct_per, 1)*10 ).astype("int8" ).values
difficulty = torch.Tensor(difficulty ).long().to(device)
 unique_tags = pd.concat([que_data.tags1,que_data.tags2, que_data.tags3, que_data.tags4,que_data.tags5,que_data.tags6] ).unique()
tags_n = len(unique_tags)
 unk_tag = tags_n-1
que_data = que_data.replace(-1, unk_tag)
 part_valus = torch.from_numpy(que_data.part.values ).long().to(device)
 que_data = que_data.to_dict("index")
 que_arr = np.zeros(( np.array(list(que_data.keys())).shape[0], 6))
for i in que_data:
a = que_data[i]
que_arr[i] = [a['tags1'],a['tags2'],a['tags3'],a['tags4'],a['tags5'],a['tags6']]
with open('.. /input/hashtable/user_info', 'rb')as handle:
user_info = pickle.load(handle )<drop_column>
submission.to_csv('submission.csv', index=False )
st_user_info = {}
for i in user_info:
st_user_info[i] = {"timestamp_ms":user_info[i]["first_timestamp"]}
del user_info<categorify>
submission.to_csv('submission.csv', index=False )