Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<choose_model_class>
!pip install knockknock
!pip install efficientnet-pytorch
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
user_secrets = UserSecretsClient()
token = user_secrets.get_secret("token")
 chat_id = user_secrets.get_secret("chat_id")
 warnings.filterwarnings('ignore' )
def model_gru_srk_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
SEED = 42
BATCH_SIZE = 8
SIZE = [520,520]
LR = 0.001
WEIGHT_DECAY = 0
EPOCHS = 35
WARMUP = 15
STEP_SIZE = 5
TTA = 4
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<find_best_params>
def seed_everything(SEED):
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(SEED)
 device = torch.device("cuda:0" )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<split>
DIR_INPUT = '/kaggle/input/plant-pathology-2020-fgvc7'
train_df = pd.read_csv(DIR_INPUT + '/train.csv')
 test_df = pd.read_csv(DIR_INPUT + '/test.csv')
 cols = list(train_df.columns[1:] )
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index )<define_variables>
class PLANT(Dataset):
def __init__(self,df,transform=None,train=True):
self.df = df
self.transform = transform
self.train = train
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
path = self.df.iloc[idx]['image_id']
image = cv2.imread(DIR_INPUT+f"/images/{path}.jpg")
 image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
 if self.transform is not None:
image = self.transform(image=image)['image']
if self.train==True:
label = np.argmax(self.df[cols].iloc[idx].values ).reshape(1,1)
 return {'image': image,'label': label }
if self.train==False:
return {'image':image }
outputs = [] 
<predict_on_test>
train,valid = train_test_split(train_df,test_size = 0.2,random_state = SEED)
 dataset_train = PLANT(df=train, transform=transform['train'])
 dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=4, shuffle=True, drop_last=True)
 dataset_valid = PLANT(df=valid, transform=transform['valid'])
 dataloader_valid = DataLoader(dataset_valid, batch_size=BATCH_SIZE, num_workers=4, shuffle=False, drop_last=True)
 dataset_test = PLANT(test_df,transform=transform['valid'],train=False)
 dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)
 dataset_test_tta = PLANT(test_df,transform=transform['test_tta'],train=False)
 dataloader_test_tta = DataLoader(dataset_test_tta, batch_size=BATCH_SIZE, num_workers=4, shuffle=False )
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_gru_srk_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'gru atten srk'] )<predict_on_test>
def getmodel() :
model = EfficientNet.from_pretrained('efficientnet-b5')
 model._fc = nn.Sequential(
 nn.Linear(in_features=2048, out_features=4, bias=True))
model = model.to(device)
 return model
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN'] )<predict_on_test>
new_model = getmodel()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(new_model.parameters() , lr=LR,weight_decay=WEIGHT_DECAY)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=WARMUP, num_training_steps=EPOCHS )
pred_val_y, pred_test_y, best_score = train_pred(model_cnn_flip(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN flip'] )<predict_on_test>
results = pd.DataFrame(columns=['training_loss','training_accuracy','validation_loss','validation_accuracy','precision','recall','roc_auc_score'])
 @telegram_sender(token=token, chat_id=int(chat_id))
def train(model, criterion, optimizer,scheduler, dataloader_train, dataloader_valid):
global results
for epoch in range(EPOCHS):
print('Epoch {}/{}'.format(epoch,EPOCHS-1))
since = time.time()
model.train()
training_accuracy = []
training_loss = []
for bi, d in enumerate(tqdm(dataloader_train, total=int(len(dataloader_train)))) :
inputs = d["image"]
labels = d["label"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 optimizer.zero_grad()
with torch.set_grad_enabled(True):
outputs = model(inputs)
 labels = labels.squeeze()
loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
outputs = torch.max(outputs,1)[1]
outputs = outputs.cpu().detach().numpy()
labels = labels.cpu().numpy()
training_accuracy.append(accuracy_score(outputs,labels))
training_loss.append(loss.item())
 print('Training accuracy: {:.4f} and Training Loss: {:.4f}'.format(np.mean(training_accuracy),np.mean(training_loss)))
 model.eval()
validation_loss = []
validation_labels = []
validation_outputs = []
with torch.no_grad() :
for bi,d in enumerate(tqdm(dataloader_valid,total=int(len(dataloader_valid)))) :
inputs = d["image"]
labels = d["label"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 outputs = model(inputs)
 labels = labels.squeeze()
loss = criterion(outputs,labels)
 outputs_softmax = F.softmax(outputs ).cpu().detach().numpy()
labels_onehot = torch.eye(4)[labels].cpu().numpy()
validation_labels.extend(labels_onehot)
 validation_outputs.extend(outputs_softmax)
 validation_loss.append(loss.item())
 precision = precision_score(np.argmax(validation_labels,axis=1),np.argmax(validation_outputs,axis=1),average='macro')
 recall = recall_score(np.argmax(validation_labels,axis=1),np.argmax(validation_outputs,axis=1),average='macro')
 accuracy = accuracy_score(np.argmax(validation_labels,axis=1),np.argmax(validation_outputs,axis=1))
roc = roc_auc_score(validation_labels,validation_outputs,average='macro')
 print('Validation accuracy: {:.4f} and Validation Loss: {:.4f} and roc_auc_score: {:.4f}'.format(accuracy,\
np.mean(validation_loss),roc))
res = pd.DataFrame([[np.mean(training_loss),np.mean(training_accuracy),np.mean(validation_loss),\
accuracy,precision,recall,roc]],columns=results.columns)
 results = pd.concat([results,res])
 scheduler.step()
return results.iloc[-1]
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN GloVe'] )<predict_on_test>
train(new_model, criterion, optimizer,scheduler, dataloader_train, dataloader_valid )
pred_val_y, pred_test_y, best_score = train_pred(model_cnn_flip(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN GloVe flip'] )<predict_on_test>
display_training_curves(1, results['precision'], 'precision', 211)
 display_training_curves(1, results['recall'], 'recall', 212 )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU'] )<predict_on_test>
new_model.eval()
test_pred = np.zeros(( len(test_df),4))
with torch.no_grad() :
for i, data in enumerate(tqdm(dataloader_test,total=int(len(dataloader_test)))) :
inputs = data['image']
inputs = inputs.to(device, dtype=torch.float)
 predict = new_model(inputs)
 test_pred[i*len(predict):(i+1)*len(predict)] = predict.detach().cpu().squeeze().numpy()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention'] )<predict_on_test>
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = F.softmax(torch.from_numpy(test_pred),dim=1)
 submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention GloVe'] )<predict_on_test>
new_model.eval()
test_pred = np.zeros(( len(test_df),4))
for i in range(TTA):
with torch.no_grad() :
for i, data in enumerate(tqdm(dataloader_test_tta,total=int(len(dataloader_test_tta)))) :
inputs = data['image']
inputs = inputs.to(device, dtype=torch.float)
 predict = new_model(inputs)
 test_pred[i*len(predict):(i+1)*len(predict)] += predict.detach().cpu().squeeze().numpy()
<sort_values><EOS> 
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = F.softmax(torch.from_numpy(test_pred/TTA),dim=1)
 submission_df.to_csv('submission_tta.csv', index=False)
 pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts()