Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<init_hyperparams>
from IPython.display import clear_output
embed_size = 300
max_features = 50000
maxlen = 100
S_DROPOUT = 0.4
DROPOUT = 0.1<import_modules>
!pip install -U efficientnet
clear_output()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
import gc
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, CuDNNLSTM, concatenate
from keras.layers import Bidirectional, GlobalMaxPool1D, Dropout, SpatialDropout1D, GlobalAveragePooling1D, GlobalMaxPooling1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<string_transform>
import efficientnet.tfkeras as efn
from matplotlib import pyplot as plt
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<statistical_test>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_0.jpg')
 print(img.shape )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
train_df = pd.read_csv("/kaggle/input/plant-pathology-2020-fgvc7/train.csv")
 test_df = pd.read_csv("/kaggle/input/plant-pathology-2020-fgvc7/test.csv")
 sub_df = pd.read_csv("/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv" )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector
del embeddings_index; gc.collect()<features_selection>
img_size = 1024
IMAGE_SIZE = [1024,1024]
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
n_classes = 4
CLASSES = 4
EMBEDDING_FILE = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_4 =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features: continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix_4[i] = embedding_vector
del embeddings_index; gc.collect()<concatenate>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
embedding_matrix = np.concatenate(( embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4
gc.collect()
np.shape(embedding_matrix )<choose_model_class>
train_tpu_paths = train_df.image_id.apply(lambda x: GCS_DS_PATH+"/images/"+x+".jpg" ).values
train_labels = train_df.iloc[:, 1:].values
test_tpu_paths = test_df.image_id.apply(lambda x: GCS_DS_PATH+"/images/"+x+".jpg" ).values
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size * 4, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(S_DROPOUT )(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 x = Dense(16, activation="relu" )(conc)
 x = Dropout(DROPOUT )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if tf.random.uniform([1])>0.4:
image = tf.image.adjust_brightness(image, 0.2)
 if label is None:
return image
else:
return image, label
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
pred_val_lstm_y = model.predict([val_X], batch_size=1024, verbose=1 )<predict_on_test>
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
pred_test_lstm_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
train_set =(tf.data.Dataset 
.from_tensor_slices(( train_tpu_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.map(transform, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO))
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_tpu_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
del all_embs, model, inp, x
time.sleep(10 )<choose_model_class>
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='noisy-student',
include_top=False 
)
 enet.trainable = True
model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy']
)
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size * 4, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(S_DROPOUT )(x)
 x = Reshape(( maxlen, embed_size * 4, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size * 4),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(DROPOUT )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
%%time
hist = model.fit(train_set,
steps_per_epoch=train_labels.shape[0]//BATCH_SIZE,
epochs = EPOCHS,
callbacks=[lr_callback],
)
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
%%time
preds = model.predict(test_dataset)
 preds
pred_val_cnn_y = model.predict([val_X], batch_size=1024, verbose=1 )<predict_on_test>
with strategy.scope() :
gc.collect()
pred_test_cnn_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
sub_df.iloc[:, 1:]=preds
time.sleep(10 )<find_best_params>
sub_df.to_csv("submission.csv", index=False )
<save_to_csv><EOS> 
su=pd.read_csv("submission.csv")
 su