Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<load_from_csv>
from sklearn.model_selection import train_test_split
import cv2
import matplotlib.pyplot as plt
import keras
from keras.preprocessing.image import img_to_array
test_image_stats = pd.read_csv(imagestatspath + "test_image_stats.csv")
 test_image_stats.head(1 )<load_from_csv>
train_df=pd.read_csv('.. /input/plant-pathology-2020-fgvc7/train.csv')
 test_df=pd.read_csv('.. /input/plant-pathology-2020-fgvc7/test.csv' )
train_image_stats_1 = pd.read_csv(imagestatspath + "train_image_stats_10000.csv")
 train_image_stats_2 = pd.read_csv(imagestatspath + "train_image_stats_20000.csv")
 train_image_stats_3 = pd.read_csv(imagestatspath + "train_image_stats_toend.csv")
 train_image_stats_4 = train_image_stats_1.append(train_image_stats_2)
 train_image_stats = train_image_stats_4.append(train_image_stats_3)
 train_image_stats.shape<define_variables>
target_df = train_df[['healthy', 'multiple_diseases', 'rust', 'scab']]
test_ids = test_df['image_id']
plot_test = True<groupby>
img_size = 300
train_imgs = []
for name in train_df['image_id']:
path = '.. /input/plant-pathology-2020-fgvc7/images/' + name + '.jpg'
img = cv2.imread(path)
 image = cv2.resize(img,(img_size,img_size),interpolation=cv2.INTER_AREA)
 train_imgs.append(image )
test_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / test_image_stats.shape[0]<groupby>
X_train = np.ndarray(shape=(len(train_imgs), img_size, img_size, 3),dtype = np.float32)
 for i, image in enumerate(train_imgs):
X_train[i] = img_to_array(image)
 X_train[i] = train_imgs[i]
X_train = X_train/255
print('Train Shape: {}'.format(X_train.shape))
train_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / train_image_stats.shape[0]<define_variables>
X_test = np.ndarray(shape=(len(test_imgs), img_size, img_size, 3),dtype = np.float32)
 for i, image in enumerate(test_imgs):
X_test[i] = img_to_array(image)
 X_test[i] = test_imgs[i]
X_test = X_test/255
print('Test Shape: {}'.format(X_test.shape))
examples1 = {"rows": 1080, "columns": 1920}
examples2 = {"rows": 4000, "columns": 6000}<load_from_csv>
y_train = train_df.copy()
del y_train['image_id']
y_train.head()
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<import_modules>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
 X_train.shape, X_val.shape
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import tensorflow.keras.applications.inception_resnet_v2 as irn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<define_variables>
lr_reduce=ReduceLROnPlateau(monitor='val_accuracy',
factor=.25,
patience=3,
min_lr=.000001,
verbose=1)
 es_monitor=EarlyStopping(monitor='val_loss',
patience=5)
 mdl_check = ModelCheckpoint('best_model.h5',
monitor='accuracy',
verbose=0,
save_best_only=True,
mode='max' )
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [256, 256, 256, 256, 256]
INC2019 = [0,0,1,1,1]
INC2018 = [1,1,0,0,0]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [0,0,0,0,0]
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
!pip install efficientnet
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
import efficientnet.keras as efn
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
net =efn.EfficientNetB3(weights='imagenet', include_top=False, input_shape=(img_size,img_size,3))
x = net.output
x = GlobalAveragePooling2D()(x)
 x = Dense(128, activation="relu" )(x)
 x = Dense(64, activation="relu" )(x)
 predictions = Dense(4, activation= 'softmax' )(x)
 model = Model(inputs = net.input, outputs = predictions )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
datagen = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
rotation_range=25,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=.1,
rescale=1/255,
fill_mode='nearest',
shear_range=0.1,
brightness_range=[0.5, 1.5] )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=16),
epochs = 14,
steps_per_epoch = X_train.shape[0] // 16,
verbose = 1,
callbacks = [es_monitor,lr_reduce, mdl_check],
validation_data = datagen.flow(X_val, y_val,batch_size=16),
validation_steps = X_val.shape[0] // 16 )
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
y_pred = model.predict(X_test )
<init_hyperparams><EOS> 
sub_df = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub_df.loc[:, 'healthy':] = y_pred
sub_df.to_csv('submission.csv', index=False)
 sub_df.head()