Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<import_modules>
resize_size = 224
%matplotlib inline
import numpy as np
import pandas as pd
import os
import json
from sklearn.cross_validation import train_test_split 
<import_modules>
np.random.seed(2019)
 tf.set_random_seed(2019 )
import gensim
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import csr_matrix, hstack, vstack
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier<set_options>
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_df.head()
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO )<prepare_x_and_y>
def get_pad_width(im, new_shape, is_rgb=True):
pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]
t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)
 l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)
 if is_rgb:
pad_width =(( t,b),(l,r),(0, 0))
else:
pad_width =(( t,b),(l,r))
return pad_width
def preprocess_image(image_path, desired_size=224):
im = Image.open(image_path)
 im = im.resize(( desired_size,)*2, resample=Image.LANCZOS)
 return im
print("Read Dataset...")
 def read_dataset(path):
	return json.load(open(path))
train = read_dataset('.. /input/train.json')
 test = read_dataset('.. /input/test.json')
 print("Prepare text data of Train and Test...")
 def generate_text(data):
	text_data = [" ".join(doc['ingredients'] ).lower() for doc in data]
	return text_data
def getSentences(data):
text_data= []
for doc in data:
text_data.append(doc['ingredients'])
 return text_data
train_text = generate_text(train)
 test_text = generate_text(test)
 Y = [doc['cuisine'] for doc in train]
_Id = [doc['id'] for doc in test]
train_sentences = getSentences(train)
 test_sentences = getSentences(test)
 sentences = train_sentences + test_sentences<choose_model_class>
N = train_df.shape[0]
x_train_rsz = np.empty(( N, resize_size, resize_size, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm_notebook(train_df['id_code'])) :
x_train_rsz[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png',desired_size = resize_size
)
tfidf = TfidfVectorizer(max_df=0.9, min_df=2 )<feature_engineering>
N = test_df.shape[0]
x_test_rsz = np.empty(( N, resize_size, resize_size, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm_notebook(test_df['id_code'])) :
x_test_rsz[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png',desired_size = resize_size
)
X = tfidf.fit_transform(train_text )<categorify>
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )
X_train_source = tfidf.transform(( ' '.join(i)for i in train_sentences))
X_test_source = tfidf.transform(( ' '.join(i)for i in test_sentences))<import_modules>
l = len(x_train)
 for j in range(2):
for i in tqdm_notebook(range(l)) :
if(y_train[i] == np.array([0,0,0,0,1])).all() :
x_train = np.vstack(( x_train, x_train[i].reshape(1,resize_size,resize_size,3)))
 y_train = np.vstack(( y_train, y_train[i].reshape(1,5)))
 print(len(x_train))
print(len(y_train))
from sklearn.model_selection import StratifiedKFold<define_variables>
print(y_train.shape)
 y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))
probaFeature = np.zeros(( X.shape[0], 40))
probaTest = []<choose_model_class>
print(x_train.shape)
 print(y_train_multi.shape)
 x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.15,
random_state=2019
)
lg = LogisticRegression(
 penalty='l2',
C=10,
n_jobs=-1, verbose=1,
solver='sag', multi_class='multinomial',
max_iter=300
)<compute_test_metric>
class MixupGenerator() :
def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):
self.X_train = X_train
self.y_train = y_train
self.batch_size = batch_size
self.alpha = alpha
self.shuffle = shuffle
self.sample_num = len(X_train)
 self.datagen = datagen
def __call__(self):
while True:
indexes = self.__get_exploration_order()
itr_num = int(len(indexes)//(self.batch_size * 2))
for i in range(itr_num):
batch_ids = indexes[i * self.batch_size * 2:(i + 1)* self.batch_size * 2]
X, y = self.__data_generation(batch_ids)
 yield X, y
def __get_exploration_order(self):
indexes = np.arange(self.sample_num)
 if self.shuffle:
np.random.shuffle(indexes)
 return indexes
def __data_generation(self, batch_ids):
_, h, w, c = self.X_train.shape
l = np.random.beta(self.alpha, self.alpha, self.batch_size)
 X_l = l.reshape(self.batch_size, 1, 1, 1)
 y_l = l.reshape(self.batch_size, 1)
 X1 = self.X_train[batch_ids[:self.batch_size]]
X2 = self.X_train[batch_ids[self.batch_size:]]
X = X1 * X_l + X2 *(1 - X_l)
 if self.datagen:
for i in range(self.batch_size):
X[i] = self.datagen.random_transform(X[i])
 X[i] = self.datagen.standardize(X[i])
 if isinstance(self.y_train, list):
y = []
for y_train_ in self.y_train:
y1 = y_train_[batch_ids[:self.batch_size]]
y2 = y_train_[batch_ids[self.batch_size:]]
y.append(y1 * y_l + y2 *(1 - y_l))
else:
y1 = self.y_train[batch_ids[:self.batch_size]]
y2 = self.y_train[batch_ids[self.batch_size:]]
y = y1 * y_l + y2 *(1 - y_l)
 return X, y
X = csr_matrix(X ).toarray() 
<choose_model_class>
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True,
brightness_range=[0.3,1.0] 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)
 mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen() )()
print(y_train[0:5] )
params = {
'multi_class': 'ovr',
'solver': 'lbfgs'
}
lgbm_params = {
'n_estimators': 250,
'max_depth': 25,
'learning_rate': 0.2,
'objective': 'multiclass',
'n_jobs': 7
}
model = LGBMClassifier(**lgbm_params)
<train_model>
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
skf = StratifiedKFold(n_splits=5, random_state=2, shuffle=False)
 for train_index, test_index in skf.split(X, Y):
print("TRAIN:", train_index, "TEST:", test_index)
 X_train, X_test = X[train_index], X[test_index]
Y_train, Y_test = Y[train_index], Y[test_index]
lg.fit(X_train, Y_train)
 probaLg = lg.predict_proba(X_test)
 score = lg.score(X_test, Y_test)
 print(score)
 model.fit(X_train, Y_train)
 probaLGBM = model.predict_proba(X_test)
 score = model.score(X_test, Y_test)
 print(score)
 probaLg_test = lg.predict_proba(X_test_source)
 probaLGBM_test = model.predict_proba(X_test_source)
 probaStack = np.hstack(( probaLg, probaLGBM))
probaTestStack = np.hstack(( probaLg_test, probaLGBM_test))
probaTest.append(probaTestStack)
 probaFeature[test_index] = probaStack<choose_model_class>
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(resize_size,resize_size,3)
 )
clfKNN = KNeighborsClassifier(
 n_neighbors=10,
p=2,
metric='minkowski',
)<define_variables>
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model
outTest = []<train_model>
model = build_model()
for train_index, test_index in skf.split(probaFeature, Y):
print("TRAIN:", train_index, "TEST:", test_index)
 X_train, X_test = probaFeature[train_index], probaFeature[test_index]
Y_train, Y_test = Y[train_index], Y[test_index]
clfKNN.fit(X_train, Y_train)
 score = clfKNN.score(X_test, Y_test)
 print(score)
 predict = clfKNN.predict_proba(probaFeatureTest)
 outTest.append(predict )<prepare_output>
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=20,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics]
)
outTest = np.array(outTest)
 outTestMean = np.mean(outTest, axis=0)
 indexClass = outTestMean.argmax(axis=1 )<define_variables>
y_val_pro=model.predict(x_val)
 print(y_val_pro[32])
 y_val_pre = y_val_pro > 0.5
print(y_val_pre[32])
 a=y_val_pre.astype(int)
 y_val_pre = y_val_pre.astype(int ).sum(axis=1)- 1
y_val_act=y_val.astype(int ).sum(axis=1)-1
diff=[]
diff=y_val_act-y_val_pre
diff_col=np.where(diff!=0)
 diff_col_ov2=np.where(np.abs(diff)>1)
 same_col=np.where(diff==0)
 QWK=cohen_kappa_score(y_val_act,y_val_pre,weights='quadratic')
 print('Accuracy=%.2f％(%.d/%.d)'%(accuracy_score(y_val_act, y_val_pre)*100,len(same_col[0]),len(y_val_act)))
 print('diff=2 is %.d'%len(diff_col_ov2[0]))
print('Cohen kappa=%.4f'%QWK )
results = []<find_best_params>
y_train_pro = model.predict(x_train)
 print("the output of CNN is the probability of diagnosis")
 print(y_train_pro[32] )
for i in indexClass:
results.append(clfKNN.classes_[i] )<save_to_csv>
clf=RFC(n_estimators=20,max_depth=10)
 y_train_act=y_train.sum(axis=1)-1
print(y_train_act)
 clf.fit(y_train_pro, y_train_act )
with open('tfidf_lg_lgbm_stack4.csv', 'w')as f:
f.write('id,cuisine
')
 for _id, y in zip(_Id, results):
f.write('%s,%s
' %(_id, y))<load_from_disk>
y_val_pre2=clf.predict(y_val_pro)
 diff=[]
diff=y_val_act-y_val_pre2
same_col=np.where(diff==0)
 print('Accuracy=%.2f％(%.d/%.d)'%(accuracy_score(y_val_act, y_val_pre2)*100,len(same_col[0]),len(y_val_act)))
 print('diff=2 is %.d'%len(diff_col_ov2[0]))
QWK=cohen_kappa_score(y_val_act,y_val_pre2,weights='quadratic')
 print('Cohen kappa=%.4f'%QWK )
train = pd.read_json('.. /input/train.json')
 test = pd.read_json('.. /input/test.json')
 gc.collect() ;<categorify>
print(y_val_pre2)
 print(y_val_act)
 print(cohen_kappa_score(y_val_act,y_val_pre2,weights='quadratic'))
y_val_pre2[3] = 3
print(y_val_pre2)
 print(y_val_act)
 print(cohen_kappa_score(y_val_act,y_val_pre2,weights='quadratic'))
procent = re.compile('[0-9]+ ?%')
 non_alphas = re.compile('[^a-z]+')
 def tok_origin(tokens):
return tokens;
def tok_clean(tokens):
cleaned = []
for token in tokens:
if len(token)<= 2:
continue
token = token.lower()
token = procent.sub('', token)
 token = non_alphas.sub(' ', token)
 cleaned.append(token.strip())
 return cleaned;<categorify>
train_data = lgb.Dataset(y_train_pro, label=y_train_act)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
'num_class': 5,
'verbose': 2,
}
gbm = lgb.train(
 params,
train_data,
num_boost_round=100,
verbose_eval=5,
)
encoder = LabelEncoder()
y_train = encoder.fit_transform(train['cuisine'] )<set_options>
preds = gbm.predict(y_val_pro)
 y_gbm_pre = []
for i in range(len(preds)) :
y_gbm_pre.append(np.argmax(preds[i]))
%%script false
parameters = {
'tfidf__tokenizer': [tok_origin, tok_clean],
'tfidf__binary': [False],
'tfidf__use_idf': [True],
'tfidf__sublinear_tf': [True],
'classifier': [
OneVsRestClassifier(MultinomialNB()),
OneVsRestClassifier(LogisticRegression(solver='saga', multi_class='multinomial'))
],
}
pipe = Pipeline(
 memory=None,
steps=[ 
('tfidf', TfidfVectorizer(preprocessor=None, lowercase=False)) , 
('classifier', OneVsRestClassifier(MultinomialNB())) ,
])
 grid = GridSearchCV(pipe, parameters, verbose=2, cv=2, iid=False, scoring='f1_micro')
 grid.fit(train['ingredients'], y_train)
 print(grid.best_params_)
 print(grid.best_score_)
<train_on_grid>
model.load_weights('model.h5')
 y_val_pred = model.predict(x_val)
 def compute_score_inv(threshold):
y1 = y_val_pred > threshold
y1 = y1.astype(int ).sum(axis=1)- 1
y2 = y_val.sum(axis=1)- 1
score = cohen_kappa_score(y1, y2, weights='quadratic')
 return 1 - score
simplex = scipy.optimize.minimize(
 compute_score_inv, 0.5, method='nelder-mead' 
)
 best_threshold = simplex['x'][0]
%%script false
parameters = {
'estimator__multi_class' : ['ovr', 'multinomial'],
'estimator__C' : [1, 5, 10, 50],
'estimator__fit_intercept' : [False, True],
'estimator__class_weight' : [None, 'balanced'],
}
model = OneVsRestClassifier(LogisticRegression(solver='saga', max_iter=500, warm_start=True, random_state=42))
grid = GridSearchCV(model, parameters, verbose=2, cv=2, iid=False, scoring='f1_micro', n_jobs=3)
 grid.fit(X_train_vec, y_train)
 print(grid.best_params_)
 print(grid.best_score_)
<set_options>
y_test = model.predict(x_test)
 y_test = gbm.predict(y_test)
 y_gbm_pre = []
for i in range(len(y_test)) :
y_gbm_pre.append(np.argmax(y_test[i]))
test_df['diagnosis'] = y_gbm_pre
test_df.to_csv('submission.csv',index=False )