Kaggle competition: APTOS 2019 Blindness Detection
roc_auc_score(Y_full.values, model.predict_proba(X_full)[:,1] )<split>
DATA_DIR = '.. /input/aptos2019-blindness-detection'
train_dir = join(DATA_DIR, 'train_images')
 label_df = pd.read_csv(join(DATA_DIR, 'train.csv'))
def train_validation_split(df, val_fraction=0.1):
val_ids = np.random.choice(df.id_code, size=int(len(df)* val_fraction))
val_df = df.query('id_code in @val_ids')
 train_df = df.query('id_code not in @val_ids')
 return train_df, val_df
train_df, val_df = train_validation_split(label_df)
 print(train_df.shape, val_df.shape)
 train_df.head()
env = riiideducation.make_env()
iter_test = env.iter_test()<merge>
%%time
class Diabetic_Retionopathy_Data(Dataset):
def __init__(self,
image_dir: str,
label_df: pd.DataFrame,
train=True,
transform=transforms.ToTensor() ,
sample_n=None,
in_memory=False,
write_images=False):

 self.image_dir = image_dir
self.transform = transform
self.train = train
self.in_memory = in_memory
if sample_n:
label_df = label_df.sample(n=min(sample_n, len(label_df)))
 ids = set(label_df.id_code)
 self.img_files = [f for f in os.listdir(image_dir)if f.split('.')[0] in ids]
label_df.index = label_df.id_code
self.label_df = label_df.drop('id_code', axis=1)
 if in_memory:
self.id2image = {}
for i, file_name in enumerate(self.img_files):
if i and i % 500 == 0:
print(f'{i} / {len(self.img_files)}')
 image = self._read_process_image(join(image_dir, file_name))
id_ = file_name.split('.')[0]
self.id2image[id_] = image
if write_images:
image.save(file_name)
 print(f'Initialized datatset with {len(self.img_files)} images.
 ')
 @staticmethod
def _read_process_image(file_path: str, size=256):
image = cv2.imread(file_path)
 return process_image_ratio_invariant(image, size=size)
 def __getitem__(self, idx):
file_name = self.img_files[idx]
id_ = file_name.split('.')[0]
if self.in_memory:
img = self.id2image[id_]
else:
img = self._read_process_image(join(self.image_dir, file_name))
X = self.transform(img)
 if self.train:
y = float(self.label_df.loc[id_].diagnosis)
 return X, y, id_
else:
return X, id_
def __len__(self):
return len(self.img_files)
 class RandomCenterCrop(transforms.CenterCrop):

 def __init__(self, min_size: int, max_size: int):
self.min_size = min_size
self.max_size = max_size
def __call__(self, img):

 size = np.random.randint(self.min_size, self.max_size + 1)
 crop = transforms.CenterCrop(( size, size))
return crop(img)
 def __repr__(self):
return f'{self.__class__.__name__}:(min-size={self.min_size}, max-size={self.max_size})'
batchsize = 16
train_transform = transforms.Compose([
RandomCenterCrop(min_size=200, max_size=256),
transforms.Resize(( 256, 256)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomRotation(( -20, 20)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 train = Diabetic_Retionopathy_Data(train_dir,
train_df,
transform=train_transform,
in_memory=True,
write_images=False)
 val = Diabetic_Retionopathy_Data(train_dir,
val_df,
transform=train_transform,
in_memory=True,
write_images=False)
 train_loader = DataLoader(train, batch_size=batchsize, num_workers=4, shuffle=True)
 val_loader = DataLoader(val, batch_size=batchsize, num_workers=3, shuffle=False)
 X, y, _ = next(iter(val_loader))
print(f'batch-dimension:
X = {X.shape},
y = {y.shape}')
 print(f'number of batches:
train: {len(train_loader)}
validation: {len(val_loader)}' )
for(test_df, sample_prediction_df)in iter_test:
test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = test_df['prior_question_had_explanation'].apply(convertBoolean)
 test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')
 test_df = test_df.merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')
 test_df = test_df.merge(bundle_answers_df, how = 'left', on = 'bundle_id')
 test_df = test_df.merge(part_answers_df, how = 'left', on = 'part')
 test_df = test_df.merge(user_lecture_stats_part, how = 'left', on=['user_id'])
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df["toeic_section"] = test_df['part'].apply(TOEICSection)
 cat_ce = count_en.fit_transform(test_df["toeic_section"])
 test_df = test_df.join(cat_ce.add_suffix("_ce"))
test_df["tags_count"] = test_df['tags'].apply(tagsCount)
 test_df['sum_answered_correctly_user'].fillna(0, inplace=True)
 test_df.fillna(0.5, inplace=True)
 target_columns = ['question_id', 'part']
test_df = test_df.join(target_enc.transform(test_df[target_columns] ).add_suffix('_target'))
catboost_columns = ['prior_question_elapsed_time', 'prior_question_had_explanation_enc']
test_df = test_df.join(catboost_enc.transform(test_df[catboost_columns] ).add_suffix('_catboost'))
test_df['answered_correctly'] = model.predict_proba(test_df[X_full.columns])[:,1]
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
def count_parameters(model: nn.Module):
return sum([np.prod(x.shape)for x in model.parameters() ])
 def print_lr_schedule(lr: float, decay: float, num_epochs=20):
print('
learning-rate schedule:')
 for i in range(num_epochs):
if i % 2 == 0:
print(f'{i}\t{lr:.6f}')
 lr = lr* decay
net = EfficientNet.from_name('efficientnet-b0')
 net.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))
num_in_features = net._fc.in_features
net._fc = nn.Linear(num_in_features, 1)
 print(f'number of parameters: {count_parameters(net)}')
 net.train()
net.cuda()
loss_function = nn.MSELoss()
lr = 0.0015
lr_decay = 0.97
optimizer = torch.optim.Adam(net.parameters() , lr=lr, weight_decay=1e-5)
 scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=lr_decay)
 print_lr_schedule(lr, lr_decay )
import optuna
import lightgbm as lgb
import riiideducation
import dask.dataframe as dd
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score<load_from_csv>
%%time
best_epoch_score = np.inf
print('epoch\ttrain-MSE\tval-MSE\tq-kappa\tlr\t\ttime [min]')
 print('------------------------------------------------------------------')
 for epoch in range(25):
start = time.time()
train_loss = []
for i,(X, y, id_)in enumerate(train_loader):
net.train()
optimizer.zero_grad()
out = net(X.cuda())
 loss = loss_function(out, y.float().cuda().view(-1, 1))
train_loss.append(loss.item())
 loss.backward()
optimizer.step()
validation_loss = []
predictions = np.array([])
 truth = np.array([])
 for X, y, id_ in val_loader:
net.eval()
out = net(X.cuda())
 loss = loss_function(out, y.float().cuda().view(-1, 1))
validation_loss.append(loss.item())
 predictions = np.append(predictions, out.detach().cpu().numpy())
 truth = np.append(truth, y.detach().cpu().numpy().astype(int))
current_lr = optimizer.param_groups[0]['lr']
scheduler.step()
qk = cohen_kappa_score(predictions.round().astype(int), truth, weights='quadratic')
 duration =(time.time() - start)/ 60
print(f'{epoch}:\t{np.mean(train_loss):.4f}\t\t{np.mean(validation_loss):.4f}\t{qk:.4f}\t{current_lr:.6f}\t{duration:.2f}')
 if np.mean(validation_loss)< best_epoch_score:
torch.save(net.state_dict() , 'state_dict_best.pt')
 best_epoch_score = np.mean(validation_loss)
 best_epoch = epoch
print(f'epoch with best validation-score: {best_epoch}')
 plt.hist(predictions, bins=5)
 plt.xlim(-1, 5)
 plt.title('distribution of predictions 
(before rounding)')
 plt.show()
plt.hist(train_df.diagnosis.values, bins=5)
 plt.xlim(-1, 5)
 plt.title('distribution of labels')
 plt.show()
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3,4,7,8,9], dtype={'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8','prior_question_elapsed_time': 'float32','prior_question_had_explanation': 'boolean'} 
)
 train = train[train.content_type_id == False]
train = train.sort_values(['timestamp'], ascending=True)
 train.drop(['timestamp','content_type_id'], axis=1, inplace=True)
 results_c = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c.columns = ["answered_correctly_content"]
results_u = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum'])
 results_u.columns = ["answered_correctly_user", 'sum']<load_from_csv>
test_dir = join(DATA_DIR, 'test_images')
 test_df = pd.read_csv(join(DATA_DIR, 'test.csv'))
test_df.head(3 )
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0,1, 3,4],
dtype={'question_id': 'int16',
'part': 'int8','bundle_id': 'int8','tags': 'str'} 
)
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
questions_df = pd.concat([questions_df,tag],axis=1)
 questions_df['tags1'] = pd.to_numeric(questions_df['tags1'], errors='coerce')
 questions_df['tags2'] = pd.to_numeric(questions_df['tags2'], errors='coerce')
 questions_df['tags3'] = pd.to_numeric(questions_df['tags3'], errors='coerce')
 questions_df['tags4'] = pd.to_numeric(questions_df['tags4'], errors='coerce')
 questions_df['tags5'] = pd.to_numeric(questions_df['tags5'], errors='coerce')
 questions_df['tags6'] = pd.to_numeric(questions_df['tags6'], errors='coerce' )<merge>
def sample_images(train_dir: str, test_dir: str, n=10):

 train_files = choice(os.listdir(train_dir), size=n)
 test_files = choice(os.listdir(test_dir), size=n)
 images = []
for train_f, test_f in zip(train_files, test_files):
train_img = Image.open(join(train_dir, train_f))
test_img = Image.open(join(test_dir, test_f))
images.append(( train_img, test_img))
def show_image(i):
train_image, test_image = images[i]
fig,(ax1, ax2)= plt.subplots(1, 2)
 fig.set_size_inches(20, 10)
 ax1.imshow(train_image)
 ax2.imshow(test_image)
 ax1.set_title('train')
 ax2.set_title('test')
 plt.show()
return interactive(show_image, i=(0, n-1))
sample_images(train_dir=train_dir,
test_dir=test_dir,
n=10)
X = train.iloc[80000000:,:]
X['prior_question_had_explanation'].fillna(False, inplace=True)
 X = pd.merge(X, results_u, on=['user_id'], how="left")
 X = pd.merge(X, results_c, on=['content_id'], how="left")
 X = pd.merge(X, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X=X[X.answered_correctly!= -1 ]
X=X.sort_values(['user_id'])
 Y = X[["answered_correctly"]]
X = X.drop(["answered_correctly"], axis=1 )<train_model>
test_transform = transforms.Compose([
transforms.Resize(( 256, 256)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomRotation(( -20, 20)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 test_ds = Diabetic_Retionopathy_Data(test_dir,
test_df,
transform=test_transform,
train=False)
 test_loader = DataLoader(test_ds, batch_size=batchsize)
 X, _ = next(iter(test_loader))
print(f'batch-dimension:
X = {X.shape}' )
print('Training data shrinked from {:,} records({:,} users)to {:,} records({:,} users ).'.format(train.shape[0], train['user_id'].nunique() , X.shape[0], X['user_id'].nunique()))<categorify>
net.load_state_dict(torch.load('state_dict_best.pt'))
net.eval()
net.cuda()
id2prediction = {}
for i,(X, id_)in enumerate(test_loader):
out = net(X.cuda())
 preds = out.detach().cpu().numpy().ravel()
id2prediction = {**id2prediction, **dict(zip(id_, preds.round().astype(int ).tolist())) }
<create_dataframe><EOS> 
submission_df = pd.read_csv(join(DATA_DIR, 'sample_submission.csv'))
submission_df.diagnosis = submission_df.id_code.map(id2prediction)
 submission_df.diagnosis = submission_df.diagnosis.map(lambda p: max(p, 0))
submission_df.diagnosis = submission_df.diagnosis.map(lambda p: min(p, 4))
submission_df.to_csv('submission.csv', index=False)
 display(submission_df.head() )