Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<compute_test_metric>
!pip install -q efficientnet
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_test_metric>
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from sklearn import metrics
from sklearn.model_selection import train_test_split
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<split>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
train_epochs = 5
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = BasicNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(weight=None, reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for j,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[j * batch_size:(j+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.15, random_state=2020 )
sub = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub['prediction'] = sub['prediction'].apply(lambda x : int(x))
sub.to_csv("./submission.csv", index=False )<import_modules>
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
ps = PorterStemmer()
def read_LSTM(reset_data, max_length):
if(reset_data or not os.path.exists('lstm_data_vector.npy')or not os.path.exists("lstm_labels_vector.npy")or not os.path.exists("lstm_qid_vector.npy")) :
train = pd.read_csv('.. /input/quora-insincere-questions-classification/train.csv' ).fillna('')
 test = pd.read_csv('.. /input/quora-insincere-questions-classification/test.csv' ).fillna('')
 qid = test.drop('question_text', axis=1)
 training_labels = train['target'].to_numpy()
train_text = train['question_text']
test_text = test['question_text']
text_list = pd.concat([train_text, test_text])
 y = train['target'].values
num_train_data = y.shape[0]
nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 else:
padded_train = np.load("lstm_data_vector.npy")
 training_labels = np.load("lstm_data_vector.npy")
 qid = np.load("lstm_qid_vector.npy")
 padded_test = np.load("lstm_test_vector.npy")
 x_train, x_test, y_train, y_test = train_test_split(train_word_sequences, training_labels, test_size=0.01, random_state=27)
 print('x_train.shape: ', x_train.shape)
 print('x_test.shape: ', x_test.shape)
 print('y_train.shape: ', y_train.shape)
 print('y_test.shape: ', y_test.shape)
 vocab_size = len(word_dict)
 print(vocab_size)
 return word_dict, lemma_dict, test_word_sequences, qid, x_train, x_test, y_train, y_test, vocab_size
def do_LSTM(word_dict, lemma_dict, actual_test, x_train, x_test, y_train, y_test, vocab_size, max_question_length):
embeddings_index={}
with zipfile.ZipFile(".. /input/quora-insincere-questions-classification/embeddings.zip")as zf:
with io.TextIOWrapper(zf.open("glove.840B.300d/glove.840B.300d.txt"), encoding="utf-8")as f:
for line in tqdm(f):
values=line.split(' ')
 word=values[0]
vectors=np.asarray(values[1:],'float32')
 embeddings_index[word]=vectors
embeddings_index = dict(embeddings_index)
 print('embedding stuff')
 embed_size = 300
embedding_matrix = np.zeros(( len(word_dict)+1, embed_size), dtype=np.float32)
 empty = np.zeros(( embed_size,), dtype=np.float32)- 1.
 for key in tqdm(word_dict):
word = key
embedding_vector1 = embeddings_index.get(word)
 word = key.lower()
embedding_vector2 = embeddings_index.get(word)
 word = key.upper()
embedding_vector3 = embeddings_index.get(word)
 word = key.capitalize()
embedding_vector4 = embeddings_index.get(word)
 word = ps.stem(key)
 embedding_vector5 = embeddings_index.get(word)
 word = lemma_dict[key]
embedding_vector6 = embeddings_index.get(word)
 if embedding_vector1 is not None:
embedding_matrix[word_dict[key]] = embedding_vector1
continue
if embedding_vector2 is not None:
embedding_matrix[word_dict[key]] = embedding_vector2
continue
if embedding_vector3 is not None:
embedding_matrix[word_dict[key]] = embedding_vector3
continue
if embedding_vector4 is not None:
embedding_matrix[word_dict[key]] = embedding_vector4
continue
if embedding_vector5 is not None:
embedding_matrix[word_dict[key]] = embedding_vector5
continue
if embedding_vector6 is not None:
embedding_matrix[word_dict[key]] = embedding_vector6
continue
embedding_matrix[word_dict[key]] = empty
model = Sequential()
model.add(Embedding(vocab_size+1, embed_size, weights=[embedding_matrix], input_length=max_question_length, trainable=False))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 model.fit(x_train, y_train, epochs=3, batch_size=512)
 print("starting predict")
 start = time.time()
predictions = model.predict_classes(x_test)
 print('Time to predict: ', time.time() - start)
 print("starting Real predict")
 start = time.time()
actual_pred = model.predict_classes(actual_test)
 print('Time to real predict: ', time.time() - start)
 return actual_pred, predictions
def main() :
max_length = 50
lstm = True
reset_data = True
if lstm:
word_dict, lemma_dict, actual_test, qid, x_train, x_test, y_train, y_test, vocab_size = read_LSTM(reset_data, max_length)
 else:
return
print("starting training")
 start = time.time()
actual_pred, tes = do_LSTM(word_dict, lemma_dict, actual_test, x_train, x_test, y_train, y_test, vocab_size, max_length)
 print('Time to train & predict: ', time.time() - start)
 print("confusion matrix:
", confusion_matrix(y_test, tes))
print("accuracy: ", accuracy_score(y_test, tes))
print("f1: ", f1_score(y_test, tes, average='macro'))
print("classification report:
", classification_report(y_test, tes))
print("precision: ", precision_score(y_test, tes))
print("recall: ", recall_score(y_test, tes))
print(actual_pred)
 qid['prediction'] = actual_pred
qid.to_csv('submission.csv', index=False)
 if __name__ == "__main__":
main()<import_modules>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<import_modules>
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
import nltk
import string
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
import re<load_from_csv>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<string_transform>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
def clean_text(text):
text = text.translate(string.punctuation)
 text = text.lower().split()
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops and len(w)>= 3]
text = " ".join(text)
 text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return text
print("Clean Text Function defined" )<feature_engineering>
probs = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()