Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<load_from_csv>
!pip install -q efficientnet
PATH = '.. /input/melanoma-oof-and-sub/'
columns_map = {
'age_approx': 'age',
'anatom_site_general_challenge': 'location',
}
def get_train() :
df = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 df['sex'].fillna('unknown', inplace=True)
 df = df.rename(columns=columns_map)
 df['location'].fillna('unknown', inplace=True)
 filenames = np.sort([f for f in os.listdir(PATH)if 'oof' in f])
 for k, filename in enumerate(filenames):
df_preds = pd.read_csv(f'{PATH}{filename}')
 df = df.merge(df_preds[['image_name', 'pred']].rename(columns={'pred': f'p_{k}'}), how='inner', on="image_name")
 if k == 0:
df['fold'] = df_preds['fold']
return df
def get_test() :
df = pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv')
 df['sex'].fillna('unknown', inplace=True)
 df = df.rename(columns=columns_map)
 df['location'].fillna('unknown', inplace=True)
 filenames = np.sort([f for f in os.listdir(PATH)if 'sub' in f])
 for k, filename in enumerate(filenames):
df_preds = pd.read_csv(f'{PATH}{filename}')
 df = df.merge(df_preds[['image_name', 'target']].rename(columns={'target': f'p_{k}'}), how='inner', on="image_name")
 return df
def add_features(df_train, df_test):
df_train = pd.concat([df_train, pd.get_dummies(df_train['sex'], prefix='sex')], axis=1)
 df_test = pd.concat([df_test, pd.get_dummies(df_test['sex'], prefix='sex')], axis=1)
 df_train = pd.concat([df_train, pd.get_dummies(df_train['location'], prefix='anatom')], axis=1)
 df_test = pd.concat([df_test, pd.get_dummies(df_test['location'], prefix='anatom')], axis=1)
 L = 15
features = ['sex','age', 'location']
M = df_train.target.mean()
te = df_train.groupby(features)['target'].agg(['mean','count'] ).reset_index()
te['ll'] =(( te['mean']*te['count'])+(M*L)) /(te['count']+L)
 del te['mean'], te['count']
df_train = df_train.merge(te, on=features, how='left')
 df_train['ll'] = df_train['ll'].fillna(M)
 df_test = df_test.merge(te, on=features, how='left')
 df_test['ll'] = df_test['ll'].fillna(M)
 return df_train, df_test<prepare_x_and_y>
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
from sklearn import metrics
from sklearn.model_selection import train_test_split
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2
from tensorflow.keras.layers import Flatten,Dense,Dropout,BatchNormalization
from tensorflow.keras.models import Model,Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
def train(df, params, features, nfolds=5, verbose_eval=10):
folds = range(nfolds)
 target = 'target'
df_oof = df[['image_name', 'target', 'fold']].copy()
y = df[target].values
oof_preds = np.zeros(( df_oof.shape[0], 1))
scores = []
for fold in folds:
print(f'==========> Training fold {fold} <==========')
 df_train = df[df.fold != fold]
df_valid = df[df.fold == fold]
valid_idx = df_valid.index
X_train, y_train = df_train[features], df_train[target]
X_valid, y_valid = df_valid[features], df_valid[target]
run_params = params.copy()
model = CatBoostClassifier(**run_params)
 model.fit(X_train, y_train,
eval_set=(X_valid, y_valid),
use_best_model=True,
early_stopping_rounds=50,
verbose_eval=10)
 y_preds = model.predict_proba(X_valid)[:,1]
score_fold = roc_auc_score(y_valid, y_preds)
 print(f'auc roc score fold {fold} auc: {score_fold}')
 scores.append(score_fold)
 oof_preds[valid_idx] = y_preds.reshape(-1,1)
 df_oof.loc[df_oof.fold == fold, "pred"] = y_preds.reshape(-1,1)
 model.save_model(f'model_{fold}')
 score = roc_auc_score(y, df_oof['pred'])
 print(f'auc roc score: {score}')
 print(f'Mean auc roc score per fold: {sum(scores)/len(scores):0.5f}')
 return df_oof<predict_on_test>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 50
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
def predict(df, modelpaths, features):
print(f'======================> Start Predictions <======================')
 df_preds = df[['image_name']].copy()
df_preds['target'] = 0.0
y_preds = np.zeros(( df.shape[0], 1))
X = df[features].values
n = len(modelpaths)
 for k, modelpath in enumerate(modelpaths):
print(f'Predicting with model {k}')
 model = CatBoostClassifier()
model.load_model(modelpath)
 y_preds += model.predict_proba(X)[:,1].reshape(-1,1)/n
df_preds['target'] = y_preds
return df_preds<save_to_csv>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
def create_submission(df_preds):
submission = pd.read_csv(".. /input/siim-isic-melanoma-classification/sample_submission.csv")
 submission.target = df_preds.target.values.reshape(-1)
 submission.to_csv(f'submissions.csv', index=False )<split>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.06, random_state=2020)
 train_paths = train.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
df_train = get_train()
df_test = get_test()
df_train, df_test = add_features(df_train, df_test )<train_model>
img_size = 768
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label
features = [
'll',
'p_1',
'p_3',
'p_8',
'p_10',
'p_12',
'p_21',
'p_26',
'p_37',
]
params = {
'loss_function': 'CrossEntropy',
'eval_metric': 'AUC',
'bootstrap_type': 'Bayesian',
'bagging_temperature': 0.5,
'grow_policy': 'SymmetricTree',
'depth': 3,
'learning_rate': 0.03,
'iterations': 2000,
}
df_oof = train(df_train, params, features, nfolds=5 )<predict_on_test>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
modelpaths = [f'model_{k}' for k in range(5)]
df_preds = predict(df_test, modelpaths, features )<compute_test_metric>
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
create_submission(df_preds )<import_modules>
def get_model(use_model):
base_model = use_model(weights='noisy-student',
include_top=False, pooling='avg',
input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(train_labels.shape[1], activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(EfficientNetB7)
 model.compile(optimizer='nadam', loss='categorical_crossentropy',metrics=['categorical_accuracy'])
 model.summary()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt<import_modules>
history = model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[lr_callback, ModelCheckpoint(filepath='pretrained_EfficientNetB7.h5', monitor='val_loss', save_best_only=True)],
validation_data=valid_dataset, epochs=EPOCHS )
from sklearn import metrics
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras.layers as L
from kaggle_datasets import KaggleDatasets
from xgboost import XGBClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import KFold
import random
from functools import reduce
import scipy 
<load_from_csv>
model = tf.keras.models.load_model('pretrained_EfficientNetB7.h5' )
<load_from_csv><EOS> 
probs = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()