Kaggle competition: APTOS 2019 Blindness Detection
class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.RandomUniform(seed=10000)
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
class AdamW(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- lr * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<choose_model_class>
np.random.seed(2019)
 tf.set_random_seed(2019 )
def LSTM_GRU(spatialdropout=0.20, rnn_units=64, weight_decay=0.07):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding' )(x_input)
 emb = SpatialDropout1D(spatialdropout, seed=1024 )(emb)
 rnn1 = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=glorot_uniform(seed=111100),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(emb)
 rnn2 = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True, kernel_initializer=glorot_uniform(seed=111000),
recurrent_initializer=Orthogonal(gain=1.0, seed=1203000)) )(rnn1)
 x = concatenate([rnn1, rnn2])
 x = GlobalMaxPooling1D()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=glorot_uniform(seed=111100))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay),)
 return model<choose_model_class>
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 train_df_aug = pd.read_csv('.. /input/moredata/external/External/Label.csv')
 print(train_df.shape)
 print(test_df.shape)
 print(train_df_aug.shape)
 train_df.head()
train_df_aug.head()
def poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04):
K.clear_session()
inp = Input(shape=(maxlen,))
embedding_layer = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False )(inp)
 embedding_layer = SpatialDropout1D(spatialdropout, seed=1024 )(embedding_layer)
 rnn_1 = Bidirectional(CuDNNGRU(gru_units, return_sequences=True,
kernel_initializer=glorot_uniform(seed=10000),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(embedding_layer)
 last = Lambda(lambda t: t[:, -1], name='last' )(rnn_1)
 maxpool = GlobalMaxPooling1D()(rnn_1)
 attn = AttentionWeightedAverage()(rnn_1)
 average = GlobalAveragePooling1D()(rnn_1)
 c = concatenate([last, maxpool, attn], axis=1)
 c = Reshape(( 3, -1))(c)
 c = Lambda(lambda x:K.sum(x, axis=1))(c)
 x = BatchNormalization()(c)
 x = Dense(200, activation='relu', kernel_initializer=glorot_uniform(seed=111000))(x)
 x = Dropout(0.2, seed=1024 )(x)
 x = BatchNormalization()(x)
 output_layer = Dense(1, activation="sigmoid", kernel_initializer=glorot_uniform(seed=111000))(x)
 model = Model(inputs=inp, outputs=output_layer)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model<choose_model_class>
def get_pad_width(im, new_shape, is_rgb=True):
pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]
t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)
 l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)
 if is_rgb:
pad_width =(( t,b),(l,r),(0, 0))
else:
pad_width =(( t,b),(l,r))
return pad_width
def preprocess_image(image_path, desired_size=224):
im = Image.open(image_path)
 im = im.resize(( desired_size,)*2, resample=Image.LANCZOS)
 return im
def BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 80, 30, 12], weight_decay=0.10):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=spatialdropout, seed=10000 )(x)
 x = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True,
kernel_initializer=glorot_uniform(seed=111000),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(x)
 x1 = Conv1D(filters=filters[0], activation='relu', kernel_size=1,
padding='same', kernel_initializer=glorot_uniform(seed=110000))(x)
 x2 = Conv1D(filters=filters[1], activation='relu', kernel_size=2,
padding='same', kernel_initializer=glorot_uniform(seed=120000))(x)
 x3 = Conv1D(filters=filters[2], activation='relu', kernel_size=3,
padding='same', kernel_initializer=glorot_uniform(seed=130000))(x)
 x4 = Conv1D(filters=filters[3], activation='relu', kernel_size=5,
padding='same', kernel_initializer=glorot_uniform(seed=140000))(x)
 x1 = GlobalMaxPool1D()(x1)
 x2 = GlobalMaxPool1D()(x2)
 x3 = GlobalMaxPool1D()(x3)
 x4 = GlobalMaxPool1D()(x4)
 c = concatenate([x1, x2, x3, x4])
 x = Dense(200, activation='relu', kernel_initializer=glorot_uniform(seed=111000))(c)
 x = Dropout(0.2, seed=10000 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid", kernel_initializer=glorot_uniform(seed=110000))(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model<choose_model_class>
IMAGE_SIZE = 224
def info_image(im):
cy = im.shape[0]//2
midline = im[cy,:]
midline = np.where(midline>midline.mean() /3)[0]
if len(midline)>im.shape[1]//2:
x_start, x_end = np.min(midline), np.max(midline)
 else:
x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10
cx =(x_start + x_end)/2
r =(x_end - x_start)/2
return cx, cy, r
def resize_image(im, augmentation=True):
cx, cy, r = info_image(im)
 scaling = IMAGE_SIZE/(2*r)
 rotation = 0
if augmentation:
scaling *= 1 + 0.3 *(np.random.rand() -0.5)
 rotation = 360 * np.random.rand()
M = cv2.getRotationMatrix2D(( cx,cy), rotation, scaling)
 M[0,2] -= cx - IMAGE_SIZE/2
M[1,2] -= cy - IMAGE_SIZE/2
return cv2.warpAffine(im,M,(IMAGE_SIZE,IMAGE_SIZE))
def singleRNN(spatialdropout=0.20, rnn_units=120, weight_decay=0.08):
K.clear_session()
inp = Input(shape=(maxlen,))
embedding_layer = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False )(inp)
 embedding_layer = SpatialDropout1D(spatialdropout, seed=1024 )(embedding_layer)
 x = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True,
kernel_initializer=glorot_uniform(seed=111000),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(embedding_layer)
 x = AttentionWeightedAverage()(x)
 x = Dense(100, kernel_initializer=glorot_uniform(seed=111000))(x)
 x = Dropout(0.12, seed=1024 )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 output_layer = Dense(1, activation="sigmoid", kernel_initializer=glorot_uniform(seed=111000))(x)
 model = Model(inputs=inp, outputs=output_layer)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model<compute_test_metric>
def toCLAHEgreen(img):
clipLimit=2.0
tileGridSize=(8, 8)
 img = np.array(img)
 green_channel = img[:, :, 1]
clahe = cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize)
 cla = clahe.apply(green_channel)
 cla=clahe.apply(cla)
 return cla
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2<train_model>
y_train = pd.get_dummies(new_train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )
f1, threshold = f1_smart(np.squeeze(Y[val_list]), np.squeeze(oof[val_list]))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))<save_to_csv>
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>threshold ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<import_modules>
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.15,
random_state=2019
)
kernel_start_time = time.time() 
<define_variables>
BATCH_SIZE = 32
x_train =
data_generator = ImageDataGenerator().flow(x_train, y_train, batch_size=BATCH_SIZE )
embed_size = 300
max_features = 120000
maxlen = 80
batch_size = 1024
n_splits = 5
MEASURE_CORRELATION = False
NORMALIZE_EMBEDDINGS = True
USE_DEV_SET = False
TRY_CAPITALS = True
TRY_UPPER = True
model_input_size = embed_size
SEED = 666666<feature_engineering>
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
def seed_everything(seed=0):
seed = SEED + seed
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_from_csv>
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
 )
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )<define_variables>
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model
puncts = [
'½', '¿', 'ï', '¸', '-', ',', '/', '"', '¨', '²', 'è', '×', '❤', '，', '↓', '▾', '↑',
'Ã', '±', ']', '·', '_', '<', '?', '⋅', '™', '~', '→', '′', '>', '≤', '€', '¥', '¼',
'¶', '@', '√', '®', '\', '…', '、', '¹', '$', '•', '!', '¯', '&', '†', ')', '・', '^',
'—', '+', '
'»', '}', '¾', '§', '=', '{', '‘', '∞', 'Ø', '°', '|', '：', '▒', 'â', 'à', ':', '(',
';', '`', '│', 'é', '*', '’', '.',<feature_engineering>
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=15,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics]
)
def add_features(df):
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df['num_words'] = df.question_text.str.count('\S+')+ 1
df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
FEATURE_NAMES = ['caps_vs_length', 'words_vs_unique', 'total_length', 'num_words']
def load_and_prec() :
train_df = df_train
test_df = df_test
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(clean_text)
 test_df["question_text"] = test_df["question_text"].apply(clean_text)
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
train = add_features(train_df)
 test = add_features(test_df)
 features = train[FEATURE_NAMES].fillna(0)
 test_features = test[FEATURE_NAMES].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 filters = '
'
tokenizer = Tokenizer(num_words=max_features, filters=filters)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
return train_X, test_X, train_y, features, test_features, tokenizer.word_index<split>
model.load_weights('model.h5')
 y_val_pred = model.predict(x_val)
 def compute_score_inv(threshold):
y1 = y_val_pred > threshold
y1 = y1.astype(int ).sum(axis=1)- 1
y2 = y_val.sum(axis=1)- 1
score = cohen_kappa_score(y1, y2, weights='quadratic')
 return 1 - score
simplex = scipy.optimize.minimize(
 compute_score_inv, 0.5, method='nelder-mead' 
)
 best_threshold = simplex['x'][0]
def add_dev_set(train_X, train_y, train_features, size=100000):
dev_X = train_X[:size]
dev_y = train_y[:size]
dev_features = train_features[:size]
train_X = train_X[size:]
train_y = train_y[size:]
train_features = train_features[size:]
return train_X, train_y, train_features, dev_X, dev_y, dev_features 
<train_model>
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )