Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<feature_engineering>
%matplotlib inline
ingredients_tr = rawdf_tr['ingredients']
ingredients_te = rawdf_te['ingredients']
def sub_match(pattern, sub_pattern, ingredients):
for i in ingredients.index.values:
for j in range(len(ingredients[i])) :
ingredients[i][j] = re.sub(pattern, sub_pattern, ingredients[i][j].strip())
 ingredients[i][j] = ingredients[i][j].strip()
re.purge()
return ingredients
def regex_sub_match(series):
p0 = re.compile(r'\s*(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\s*[^a-z]')
 series = sub_match(p0, ' ', series)
 p1 = re.compile(r'\d+')
 series = sub_match(p1, ' ', series)
 p2 = re.compile('[^\w]')
 series = sub_match(p2, ' ', series)
 return series<feature_engineering>
np.random.seed(2019)
 tf.set_random_seed(2019 )
ingredients_tr = regex_sub_match(ingredients_tr )<feature_engineering>
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_df.head()
lemmatizer = WordNetLemmatizer()
def lemma(series):
for i in series.index.values:
for j in range(len(series[i])) :
series[i][j] = series[i][j].strip()
token = TK(series[i][j])
 for k in range(len(token)) :
token[k] = lemmatizer.lemmatize(token[k])
 token = ' '.join(token)
 series[i][j] = token
return series<normalization>
def get_pad_width(im, new_shape, is_rgb=True):
pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]
t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)
 l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)
 if is_rgb:
pad_width =(( t,b),(l,r),(0, 0))
else:
pad_width =(( t,b),(l,r))
return pad_width
def preprocess_image(image_path, desired_size=224):
im = Image.open(image_path)
 im = im.resize(( desired_size,)*2, resample=Image.LANCZOS)
 return im
ingredients_tr = lemma(ingredients_tr )<feature_engineering>
N = train_df.shape[0]
x_train = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(train_df['id_code'])) :
x_train[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png'
)
rawdf_tr['ingredients_lemma'] = ingredients_tr
rawdf_tr['ingredients_lemma_string'] = [' '.join(_ ).strip() for _ in rawdf_tr['ingredients_lemma']]<split>
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png'
)
def tt_split(cuisine):
cuisine_population = rawdf_tr.loc[(rawdf_tr['cuisine'] == cuisine)].index.values
train, test = train_test_split(cuisine_population, test_size=0.15, random_state=0)
 train = train.tolist()
test = test.tolist()
return train, test
cuisine_list = rawdf_tr['cuisine'].unique().tolist()
ix_train = []
ix_test = []
for _ in cuisine_list:
temp_train, temp_test = tt_split(_)
 ix_train += temp_train
ix_test += temp_test
traindf = rawdf_tr[['cuisine', 'ingredients_lemma_string']].loc[ix_train].reset_index(drop=True)
 print(traindf.shape)
 testdf = rawdf_tr[['cuisine', 'ingredients_lemma_string']].loc[ix_test].reset_index(drop=True)
 print(testdf.shape)
 X_train_ls = traindf['ingredients_lemma_string']
vectorizertr = TfidfVectorizer(stop_words='english', analyzer="word", max_df=0.65, min_df=2, binary=True)
 X_train = vectorizertr.fit_transform(X_train_ls)
 y_train = traindf['cuisine']
le = LE()
y_train_ec = le.fit_transform(y_train)
 y_train_1h = pd.get_dummies(y_train_ec)
 X_pred_ls = testdf['ingredients_lemma_string']
vectorizerts = TfidfVectorizer(stop_words='english')
 X_pred = vectorizertr.transform(X_pred_ls)
 y_true = testdf['cuisine']
y_true_ec = le.fit_transform(y_true)
 y_true_1h = pd.get_dummies(y_true_ec )<train_model>
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )
model = Sequential()
model.add(Dense(2161, input_shape=(2161,), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, activation='softsign'))
model.add(Dropout(0.58))
model.add(Dense(256, activation='softsign'))
model.add(Dropout(0.67))
model.add(Dense(20, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 epochs = 20
early_stopping = EarlyStopping(monitor='val_loss', patience=1)
 model.fit(X_train,y_train_1h,
batch_size=30,
epochs=epochs,
verbose=2,
callbacks=[early_stopping],
validation_data=(X_pred, y_true_1h),
shuffle=True )<normalization>
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))
ingredients_te = lemma(ingredients_te )<feature_engineering>
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.15,
random_state=2019
)
rawdf_te['ingredients_lemma'] = ingredients_te
rawdf_te['ingredients_lemma_string'] = [' '.join(_ ).strip() for _ in rawdf_te['ingredients_lemma']]<feature_engineering>
class MixupGenerator() :
def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):
self.X_train = X_train
self.y_train = y_train
self.batch_size = batch_size
self.alpha = alpha
self.shuffle = shuffle
self.sample_num = len(X_train)
 self.datagen = datagen
def __call__(self):
while True:
indexes = self.__get_exploration_order()
itr_num = int(len(indexes)//(self.batch_size * 2))
for i in range(itr_num):
batch_ids = indexes[i * self.batch_size * 2:(i + 1)* self.batch_size * 2]
X, y = self.__data_generation(batch_ids)
 yield X, y
def __get_exploration_order(self):
indexes = np.arange(self.sample_num)
 if self.shuffle:
np.random.shuffle(indexes)
 return indexes
def __data_generation(self, batch_ids):
_, h, w, c = self.X_train.shape
l = np.random.beta(self.alpha, self.alpha, self.batch_size)
 X_l = l.reshape(self.batch_size, 1, 1, 1)
 y_l = l.reshape(self.batch_size, 1)
 X1 = self.X_train[batch_ids[:self.batch_size]]
X2 = self.X_train[batch_ids[self.batch_size:]]
X = X1 * X_l + X2 *(1 - X_l)
 if self.datagen:
for i in range(self.batch_size):
X[i] = self.datagen.random_transform(X[i])
 X[i] = self.datagen.standardize(X[i])
 if isinstance(self.y_train, list):
y = []
for y_train_ in self.y_train:
y1 = y_train_[batch_ids[:self.batch_size]]
y2 = y_train_[batch_ids[self.batch_size:]]
y.append(y1 * y_l + y2 *(1 - y_l))
else:
y1 = self.y_train[batch_ids[:self.batch_size]]
y2 = self.y_train[batch_ids[self.batch_size:]]
y = y1 * y_l + y2 *(1 - y_l)
 return X, y
X_pred_ls = testdf['ingredients_lemma_string']
vectorizerts = TfidfVectorizer(stop_words='english')
 X_pred = vectorizertr.transform(X_pred_ls)
<predict_on_test>
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)
 mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen() )()
y_pred_nn = le.inverse_transform(model.predict_classes(X_pred))<save_to_csv>
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
testdf['cuisine'] = y_pred_nn
d = pd.DataFrame(data=testdf['cuisine'], index=testdf.index ).sort_index().reset_index().to_csv('submission_nn.csv', index=False )<set_options>
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
 )
init_notebook_mode(connected=True)
 %matplotlib inline<load_from_disk>
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model
df_train = pd.read_json('.. /input/train.json')
 df_test = pd.read_json('.. /input/test.json' )<randomize_order>
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=15,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics]
)
def get_random_colors(n):
lst = []
for i in range(n):
r = lambda: random.randint(0,255)
 lst.append('
return lst<feature_engineering>
model.load_weights('model.h5')
 y_val_pred = model.predict(x_val)
 def compute_score_inv(threshold):
y1 = y_val_pred > threshold
y1 = y1.astype(int ).sum(axis=1)- 1
y2 = y_val.sum(axis=1)- 1
score = cohen_kappa_score(y1, y2, weights='quadratic')
 return 1 - score
simplex = scipy.optimize.minimize(
 compute_score_inv, 0.5, method='nelder-mead' 
)
 best_threshold = simplex['x'][0]
<categorify><EOS> 
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )