Kaggle competition: APTOS 2019 Blindness Detection
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )<groupby>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 512
NUM_CLASSES = 5
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']<groupby>
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 x = df_train['id_code']
y = df_train['diagnosis']
x, y = shuffle(x, y, random_state=8 )
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )<groupby>
y = to_categorical(y, num_classes=NUM_CLASSES)
 train_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,
stratify=y, random_state=8)
 print(train_x.shape)
 print(train_y.shape)
 print(valid_x.shape)
 print(valid_y.shape)
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<set_options>
class My_Generator(Sequence):
def __init__(self, image_filenames, labels, batch_size, is_train=True, mix=False):
self.image_filenames, self.labels = image_filenames, labels
self.batch_size = batch_size
self.is_train = is_train
self.on_epoch_end()
self.is_mix = mix
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
return self.train_generate(batch_x, batch_y)
 return self.valid_generate(batch_x, batch_y)
 def on_epoch_end(self):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 def mix_up(self, x, y):
lam = np.random.beta(0.2, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def train_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/'+sample+'.png')
 img = cv2.resize(img,(SIZE, SIZE))
img = seq.augment_image(img)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 if(self.is_mix):
batch_images, batch_y = self.mix_up(batch_images, batch_y)
 return batch_images, batch_y
def valid_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/'+sample+'.png')
 img = cv2.resize(img,(SIZE, SIZE))
batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 return batch_images, batch_y
del(train)
 gc.collect()<merge>
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D)
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )<merge>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model_2 = InceptionResNetV2(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model_2.load_weights('.. /input/inceptionresnetv2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = GlobalAveragePooling2D()(base_model_2.output)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )<categorify>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 15; batch_size = 16
checkpoint = ModelCheckpoint('.. /working/combo.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=7)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early]
train_generator = My_Generator(train_x, train_y, batch_size, is_train=True)
 train_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=True)
 valid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)
 model = create_model(
 input_shape=(SIZE,SIZE,3),
n_out=NUM_CLASSES )
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<categorify>
for layer in model.layers:
layer.trainable = False
for i in range(-1,0):
model.layers[i].trainable = True
model.compile(
 loss='categorical_crossentropy',
optimizer='sgd')
 model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_y)) / float(128)) ,
epochs=7,
class_weight=class_weight,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
verbose=1 )
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )<merge>
for layer in model.layers:
layer.trainable = True
callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=1e-4),
metrics=['accuracy'])
 model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)) ,
validation_data=valid_generator,
validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)) ,
class_weight=class_weight,
epochs=epochs,
verbose=1,
max_queue_size=16, workers=2, use_multiprocessing=True,
callbacks=callbacks_list )
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1<prepare_x_and_y>
submit = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 model.load_weights('.. /working/combo.h5')
 predicted = []
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<drop_column>
for i, name in tqdm(enumerate(submit['id_code'])) :
path = os.path.join('.. /input/aptos2019-blindness-detection/test_images/', name+'.png')
 image = cv2.imread(path)
 image = cv2.resize(image,(SIZE, SIZE))
score_predict = model.predict(( image[np.newaxis])/255)
 label_predict = np.argmax(score_predict)
 predicted.append(str(label_predict))
<data_type_conversions><EOS> 
submit['diagnosis'] = predicted
submit.to_csv('submission.csv', index=False)
 submit.head()