Kaggle competition: Plant Pathology 2020 - FGVC7
print("Start Cleaning Data texts")
 train_df["question_text"] = train_df["question_text"].map(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].map(lambda x: clean_text(x))
print("Data Questions Text Cleaned" )<split>
!pip install -q efficientnet
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<statistical_test>
train_df = pd.read_csv(".. /input/plant-pathology-2020-fgvc7/train.csv")
 IMAGE_PATH = ".. /input/plant-pathology-2020-fgvc7/images/"
print("Creating Model by GLOVE Embeddings")
 EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
print(train_df['healthy'].value_counts())
 print(train_df['multiple_diseases'].value_counts())
 print(train_df['rust'].value_counts())
 print(train_df['scab'].value_counts() )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<find_best_params>
healthy = list(train_df[train_df["healthy"]==1].image_id)
 multiple_diseases = list(train_df[train_df["multiple_diseases"]==1].image_id)
 rust = list(train_df[train_df["rust"]==1].image_id)
 scab = list(train_df[train_df["scab"]==1].image_id )
print("Starting Prediction by GLOVE Embeddings for VAL")
 pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score<train_model>
def load_image(filenames):
sample = random.choice(filenames)
 image = load_img(".. /input/plant-pathology-2020-fgvc7/images/"+sample+".jpg")
 plt.imshow(image )
print()
print("Best Tresh for GLOVE is {0} at score {1}".format(best_thresh,best_score))
print()
pred_glove_val_y =(pred_glove_val_y>best_thresh )<predict_on_test>
load_image(healthy )
print("Prediction for test based on GLOVE Embeddings")
 pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_glove_test_y =(pred_glove_test_y>best_thresh)
 print("Glove Emebddings Prediction Complete" )<set_options>
load_image(multiple_diseases )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
load_image(rust )
print("Creating Model by Wikinews Embeddings")
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
load_image(scab )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<find_best_params>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
!gsutil ls $GCS_DS_PATH
print("Starting Prediction by wikinews Embeddings for VAL")
 pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score<compute_test_metric>
def format_path_gcs(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
X = train_df.image_id.apply(format_path_gcs ).values
y = np.float32(train_df.loc[:, 'healthy':'scab'].values)
 X_train, X_val, y_train, y_val =train_test_split(X, y, test_size=0.1, random_state=43)
 print('done!' )
print()
print("Best Tresh for wikinews is {0} at score {1}".format(best_thresh,best_score))
print()
pred_fasttext_val_y =(pred_fasttext_val_y>best_thresh )<predict_on_test>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
print("Prediction for test based on wikinews Embeddings")
 pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_fasttext_test_y =(pred_fasttext_test_y>best_thresh)
 print("Wikinews Emebddings Prediction Complete" )<set_options>
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
STEPS_PER_EPOCH = y_train.shape[0] // BATCH_SIZE
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<feature_engineering>
def decode_image(filename, label=None, image_size=(1024,1024)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_brightness(image, max_delta=0.3)
 if label is None:
return image
else:
return image, label
print("Creating Model by paragram embeddings")
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
train_dataset =(
 tf.data.Dataset.from_tensor_slices(( X_train, y_train)) 
.map(decode_image,num_parallel_calls=AUTO)
.map(data_augment,num_parallel_calls=AUTO)
.repeat() 
.shuffle(256)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_val, y_val)) 
.map(decode_image,num_parallel_calls=AUTO)
.cache() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<find_best_params>
def build_lrfn(lr_start=0.00001, lr_max=0.00005,lr_min=0.00001, lr_rampup_epochs=5,lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)*\
lr_exp_decay**(epoch - lr_rampup_epochs- lr_sustain_epochs)+ lr_min
return lr
return lrfn
print("Starting Prediction by paragram Embeddings")
 pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score<compute_test_metric>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 EarlyStopping=tf.keras.callbacks.EarlyStopping(monitor="val_loss",patience=10,verbose=True, mode="min" )
print()
print("Best Tresh for paragram is {0} at score {1}".format(best_thresh,best_score))
print()
pred_paragram_val_y =(pred_paragram_val_y>best_thresh )<predict_on_test>
def Eff_B7_NS() :
model_EfficientNetB7 = Sequential([efn.EfficientNetB7(input_shape=(1024,1024,3),weights='noisy-student',include_top=False),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(128,activation='relu'),
tf.keras.layers.Dense(64,activation='relu'),
tf.keras.layers.Dense(4,activation='softmax')])
 model_EfficientNetB7.compile(optimizer='Adam',loss = 'categorical_crossentropy',metrics=['categorical_accuracy'])
 return model_EfficientNetB7
print("Prediction for test based on paragram Embeddings")
 pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_paragram_test_y =(pred_paragram_test_y>best_thresh)
 print("Paragram Emebddings Prediction Complete" )<set_options>
EfficientNetB7 = model_Eff_B7.fit(train_dataset,
epochs=20,
callbacks=[lr_schedule,EarlyStopping],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<compute_test_metric>
TEST_PATH = ".. /input/plant-pathology-2020-fgvc7/test.csv"
SUB_PATH = ".. /input/plant-pathology-2020-fgvc7/sample_submission.csv"
sub = pd.read_csv(SUB_PATH)
 test_data = pd.read_csv(TEST_PATH )
pred_val_y = 0.34*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.33*pred_paragram_val_y
best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score<compute_test_metric>
test_paths = test_data.image_id.apply(format_path_gcs ).values
print()
print("Best Tresh for overall is {0} at score {1}".format(best_thresh,best_score))
print()
pred_val_y =(pred_val_y>best_thresh )<save_to_csv>
test_dataset =(
 tf.data.Dataset.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
<train_model><EOS> 
prediction = model_Eff_B7.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = prediction
sub.to_csv('submission.csv', index=False)
 sub.head()