Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<load_from_csv>
!pip install efficientnet > /dev/null
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
SEED = 420
def decode_image(filename, label=None, image_size=None):
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
if image_size:
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=SEED):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label
def show_batch(ds, labels):
row = 6; col = 4;
row = min(row, BATCH_SIZE//col)
 for(img, label)in ds:
plt.figure(figsize=(15,int(15*row/col)))
 for j in range(row*col):
plt.subplot(row,col,j+1)
 plt.axis('off')
 plt.imshow(img[j,])
 plt.title(labels[label[j,].numpy().argmax() ])
 plt.show()
break
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100<prepare_x_and_y>
AUTO = tf.data.experimental.AUTOTUNE
def get_strategy() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 return strategy
strategy = get_strategy()
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values<string_transform>
LABELS = [
'healthy', 'Cedar_apple_rust', 'Black_rot', 'Apple_scab',
]
IMG_PATH = Path('.. /input/plantvillageapplecolor' )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X )<string_transform>
all_img_ids = []
all_img_labels = []
for l in LABELS:
img_ids = [
'/'.join(l.parts[-2:])
 for l in(IMG_PATH /('Apple___' + l)).iterdir()
]
all_img_ids.extend(img_ids)
 all_img_labels.extend([l] * len(img_ids))
train_df = pd.DataFrame({'img_id': all_img_ids, 'img_label': all_img_labels})
 train_df.img_label = pd.Categorical(train_df.img_label, categories=LABELS, ordered=False)
 train_df = pd.concat([train_df, pd.get_dummies(train_df.img_label)], axis=1)
 train_df = train_df.sample(frac=1., random_state=SEED )
train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen )<prepare_x_and_y>
NB_CLASSES = 4
IMG_SIZE = 256
EPOCHS = 40
BATCH_SIZE = 128 * strategy.num_replicas_in_sync
N_FOLDS = 5
train_y = train_df['target'].values
val_y = val_df['target'].values<feature_engineering>
GCS_DS_PATH = KaggleDatasets().get_gcs_path('plantvillageapplecolor' )
embeddings_index = {}
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))<define_variables>
train_labels = train_df[LABELS].values.astype(np.int64)
 train_paths = train_df.img_id.apply(lambda x: GCS_DS_PATH + '/' + x ).values
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(partial(decode_image, image_size=(IMG_SIZE, IMG_SIZE)) , num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]<feature_engineering>
show_batch(train_dataset, LABELS )
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
def get_model() :
base_model = efn.EfficientNetB7(weights='imagenet', include_top=False, pooling='avg', input_shape=(None, None, 3))
x = base_model.output
predictions = Dense(NB_CLASSES, activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions )
model=Sequential()
model.add(Embedding(max_features, embed_size, weights=[embedding_matrix],input_length=maxlen,trainable = False))
model.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))
 model.add(GlobalMaxPool1D())
 model.add(Dense(32, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(1, activation="sigmoid"))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
strategy = get_strategy()
with strategy.scope() :
model = get_model()
model.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=[tf.keras.metrics.AUC() ])
 model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)],
epochs=EPOCHS
)
model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))<predict_on_test>
model.save("plant_village_pretrain_b7.h5" )
test_y_prediction = model.predict([test_X], batch_size=1024, verbose=1 )<predict_on_test>
clear_session()
del model
del train_dataset
del train_labels
del strategy
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.701, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<save_to_csv>
gc.collect()
pred_test_y =(test_y_prediction>0.32 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<set_options>
strategy = get_strategy()
np.random.seed(42)
 warnings.filterwarnings('ignore' )<load_from_csv>
NB_CLASSES = 4
IMG_SIZE = 768
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
LABEL_COLS = ['healthy', 'multiple_diseases', 'rust', 'scab']
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )<string_transform>
GCS_DS_PATH = KaggleDatasets().get_gcs_path('plant-pathology-2020-fgvc7' )
X_train = train["question_text"].fillna("fillna" ).values
y_train = train["target"].values
X_test = test["question_text"].fillna("fillna" ).values
max_features = 40000
maxlen = 70
embed_size = 300
threshold = 0.35
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )<feature_engineering>
path = '.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<predict_on_test>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(partial(decode_image, image_size=(IMG_SIZE, IMG_SIZE)) , num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
class F1Evaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 y_pred =(y_pred > threshold ).astype(int)
 score = f1_score(self.y_val, y_pred)
 print("
F1 Score - epoch: %d - score: %.6f
" %(epoch+1, score))<choose_model_class>
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(partial(decode_image, image_size=(IMG_SIZE, IMG_SIZE)) , num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
filter_sizes = [1,2,3,5]
num_filters = 42
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embed_size),
kernel_initializer='he_normal', activation='tanh' )(x)
 maxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)
 maxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)
 maxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)
 maxpool_3 = MaxPool2D(pool_size=(maxlen - filter_sizes[3] + 1, 1))(conv_3)
 z = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()<train_model>
show_batch(train_dataset, LABEL_COLS )
batch_size = 256
epochs = 2
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95,
random_state=233)
 F1_Score = F1Evaluation(validation_data=(X_val, y_val), interval=1)
 hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,
validation_data=(X_val, y_val),
callbacks=[F1_Score], verbose=2 )<save_to_csv>
def get_model() :
base_model = load_model('./plant_village_pretrain_b7.h5', compile=False)
 x = base_model.layers[-2].output
predictions = Dense(NB_CLASSES, activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions )
y_pred = model.predict(x_test, batch_size=1024)
 y_pred =(y_pred > threshold ).astype(int)
 submission['prediction'] = y_pred
submission.to_csv('submission.csv', index=False )<define_variables>
with strategy.scope() :
model = get_model()
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', tf.keras.metrics.AUC() ] )
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
input_cols = ['sequence', 'structure', 'predicted_loop_type']
error_cols = ['reactivity_error', 'deg_error_Mg_pH10', 'deg_error_Mg_50C', 'deg_error_pH10', 'deg_error_50C']
token_dicts = {
"sequence": {x: i for i, x in enumerate("ACGU")},
"structure": {x: i for i, x in enumerate('().')},
"predicted_loop_type": {x: i for i, x in enumerate("BEHIMSX")}
}
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed )<define_variables>
%%time
model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[lr_callback],
epochs=EPOCHS
)
BASE_PATH = "/kaggle/input/stanford-covid-vaccine"
MODEL_SAVE_PATH = "/kaggle/model"
def preprocess_inputs(df, cols):
return np.concatenate([preprocess_feature_col(df, col)for col in cols], axis=2)
 def preprocess_feature_col(df, col):
dic = token_dicts[col]
dic_len = len(dic)
 seq_length = len(df[col][0])
 ident = np.identity(dic_len)
 arr = np.array(
 df[[col]].applymap(lambda seq: [ident[dic[x]] for x in seq] ).values.tolist()
).squeeze(1)
 assert arr.shape ==(len(df), seq_length, dic_len)
 return arr
def preprocess(base_data, is_test=False):
inputs = preprocess_inputs(base_data, input_cols)
 if is_test:
labels = None
else:
labels = np.array(base_data[target_cols].values.tolist() ).transpose(( 0, 2, 1))
assert labels.shape[2] == len(target_cols)
 assert inputs.shape[2] == 14
return inputs, labels
def get_bpp_feature(bpp):
bpp_nb_mean = 0.077522
bpp_nb_std = 0.08914
bpp_max = bpp.max(-1)[0]
bpp_sum = bpp.sum(-1)
 bpp_nb = torch.true_divide(( bpp > 0 ).sum(dim=1), bpp.shape[1])
 bpp_nb = torch.true_divide(bpp_nb - bpp_nb_mean, bpp_nb_std)
 return [bpp_max.unsqueeze(2), bpp_sum.unsqueeze(2), bpp_nb.unsqueeze(2)]
@functools.lru_cache(5000)
 def load_from_id(id_):
path = Path(BASE_PATH)/ f"bpps/{id_}.npy"
data = np.load(str(path))
return data
def get_distance_matrix(leng):
idx = np.arange(leng)
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1 / Ds
Ds = Ds[None, :, :]
Ds = np.repeat(Ds, 1, axis=0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis=3)
 print(Ds.shape)
 return Ds
def get_structure_adj(df):
Ss = []
for i in range(len(df)) :
seq_length = df["seq_length"].iloc[i]
structure = df["structure"].iloc[i]
sequence = df["sequence"].iloc[i]
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss
def create_loader(df, batch_size=1, is_test=False):
features, labels = preprocess(df, is_test)
 features_tensor = torch.from_numpy(features)
 if labels is not None:
labels_tensor = torch.from_numpy(labels)
 dataset = VacDataset(features_tensor, df, labels_tensor)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True, drop_last=False)
 else:
dataset = VacDataset(features_tensor, df, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False, drop_last=False)
 return loader
class VacDataset(Dataset):
def __init__(self, features, df, labels=None):
self.features = features
self.labels = labels
self.test = labels is None
self.ids = df["id"]
self.score = None
self.structure_adj = get_structure_adj(df)
 self.distance_matrix = get_distance_matrix(self.structure_adj.shape[1])
 if "score" in df.columns:
self.score = df["score"]
else:
df["score"] = 1.0
self.score = df["score"]
self.signal_to_noise = None
if not self.test:
self.signal_to_noise = df["signal_to_noise"]
assert self.features.shape[0] == self.labels.shape[0]
else:
assert self.ids is not None
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
bpp = torch.from_numpy(load_from_id(self.ids[index] ).copy() ).float()
adj = self.structure_adj[index]
distance = self.distance_matrix[0]
bpp = np.concatenate([bpp[:, :, None], adj, distance], axis=2)
 if self.test:
return dict(sequence=self.features[index].float() , bpp=bpp, ids=self.ids[index])
 else:
return dict(sequence=self.features[index].float() , bpp=bpp,
label=self.labels[index], ids=self.ids[index],
signal_to_noise=self.signal_to_noise[index],
score=self.score[index])
<choose_model_class>
%%time
probs = model.predict(test_dataset )
<load_from_disk><EOS> 
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()