Kaggle competition: Cassava Leaf Disease Classification
model = ToxicBaseClassifier(
 text_field_embedder=word_embeddings,
seq2seq_encoder=None,
seq2vec_encoder=seq2vec_encoder,
dropout=0.5,
num_labels=6,
vocab=vocab 
)
 model.cuda()
trainer = Trainer(
 model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=10 
)
 metrics = trainer.train()<train_model>
CONFIG_NAME = 'stacking12.yml'
debug = False
STAGE2_DIR = '.. /input/train-stacking-2dcnn-ver3/output'
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))<categorify>
CONFIG_PATH = f'{STAGE2_DIR}/{CONFIG_NAME}'
with open(CONFIG_PATH)as f:
config = yaml.load(f)
 INFO = config['info']
TAG = config['tag']
CFG = config['cfg']
OUTPUT_DIR = './'
DATA_PATH = '.. /input/cassava-leaf-disease-classification'
seq2vec_encoder = SWEMEncoder(embedding_dim=word_embeddings.get_output_dim() )<choose_model_class>
model = ToxicBaseClassifier(
 text_field_embedder=word_embeddings,
seq2seq_encoder=None,
seq2vec_encoder=seq2vec_encoder,
dropout=0.5,
num_labels=6,
vocab=vocab 
)
 model.cuda()
trainer = Trainer(
 model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=10 
)
 metrics = trainer.train()<train_model>
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
IAAAdditiveGaussianNoise, Transpose, CenterCrop 
)
 warnings.filterwarnings('ignore')
 if CFG['debug']:
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 else:
device = torch.device('cuda')
 start_time = datetime.datetime.now()
start_time_str = start_time.strftime('%m%d%H%M' )
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))<categorify>
train = pd.read_csv(f'{DATA_PATH}/train.csv')
 test = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')
 label_map = pd.read_json(f'{DATA_PATH}/label_num_to_disease_map.json',
orient='index')
 if CFG['debug']:
train = train.sample(n=1000, random_state=CFG['seed'] ).reset_index(drop=True )
lstm = torch.nn.LSTM(
 bidirectional=True,
input_size=word_embeddings.get_output_dim() ,
hidden_size=40,
num_layers=2,
batch_first=True 
)
 seq2seq_encoder = PytorchSeq2SeqWrapper(lstm)
 seq2vec_encoder = SWEMEncoder(embedding_dim=seq2seq_encoder.get_output_dim() )<choose_model_class>
model_dirs = []
for stage1 in CFG['stage1_models']:
num = str(stage1 ).rjust(2, '0')
 output_dir_ = glob.glob(f'.. /input/{num}*/')
 assert len(output_dir_)== 1, output_dir_
model_dirs.append(output_dir_[0])
 model_dirs
model = ToxicBaseClassifier(
 text_field_embedder=word_embeddings,
seq2seq_encoder=seq2seq_encoder,
seq2vec_encoder=seq2vec_encoder,
dropout=0.5,
num_labels=6,
vocab=vocab 
)
 model.cuda()
trainer = Trainer(
 model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=10 
)
 metrics = trainer.train()<train_model>
normal_configs = []
tta_configs = []
normal_model_dirs = []
tta_model_dirs = []
for model_dir in model_dirs:
assert len(glob.glob(f'{model_dir}/*.yml')) ==1
config_path = glob.glob(f'{model_dir}/*.yml')[0]
with open(config_path)as f:
config = yaml.load(f)
 if 'valid_augmentation' in config['tag'].keys() :
tta_model_dirs.append(model_dir)
 tta_configs.append(config)
 else:
normal_model_dirs.append(model_dir)
 normal_configs.append(config )
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))<load_pretrained>
def get_score(y_true, y_pred):
return accuracy_score(y_true, y_pred)
 def remove_glob(pathname, recursive=True):
for p in glob.glob(pathname, recursive=recursive):
if os.path.isfile(p):
os.remove(p)
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
BERT_MODEL_PATH = '.. /input/bertpretrained/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
WORK_DIR = ".. /working/"
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(
 BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin' 
)
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )<load_pretrained>
TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'
token_indexer = PretrainedBertIndexer(
 pretrained_model=BERT_MODEL_PATH,
max_pieces=128,
do_lowercase=True, 
)
 tokenizer = WordTokenizer(word_splitter=BertBasicWordSplitter())
 reader = ToxicCommentClassificationReader(
 tokenizer=tokenizer,
token_indexers={"bert": token_indexer} 
)
 all_dataset = reader.read('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 train_dataset, validation_dataset = train_test_split(all_dataset, test_size=0.2, random_state=11)
 iterator = BucketIterator(
 batch_size=64,
sorting_keys=[("tokens", "num_tokens")], 
)
 vocab = Vocabulary()
iterator.index_with(vocab)
 model = ToxicBertClassifier(
 vocab=vocab,
bert_model=WORK_DIR,
num_labels=6
)<choose_model_class>
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['image_id'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image
class TTADataset(Dataset):
def __init__(self, df, image_path, ttas):
self.df = df
self.file_names = df['image_id'].values
self.labels = df['label'].values
self.image_path = image_path
self.ttas = ttas
def __len__(self)-> int:
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{self.image_path}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 imglist=[tta(image=image)['image'] for tta in self.ttas]
image=torch.stack(imglist)
 label = torch.tensor(self.labels[idx] ).long()
return image, label
model.cuda()
trainer = Trainer(model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=3)
 metrics = trainer.train()<train_model>
def _get_augmentations(aug_list, cfg):
process = []
for aug in aug_list:
if aug == 'Resize':
process.append(Resize(cfg['size'], cfg['size']))
elif aug == 'RandomResizedCrop':
process.append(RandomResizedCrop(cfg['size'], cfg['size']))
elif aug == 'CenterCrop':
process.append(CenterCrop(CFG['size'], CFG['size']))
elif aug == 'Transpose':
process.append(Transpose(p=0.5))
elif aug == 'HorizontalFlip':
process.append(HorizontalFlip(p=0.5))
elif aug == 'VerticalFlip':
process.append(VerticalFlip(p=0.5))
elif aug == 'ShiftScaleRotate':
process.append(ShiftScaleRotate(p=0.5))
elif aug == 'Normalize':
process.append(Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
))
else:
raise ValueError(f'{aug} is not suitable')
 process.append(ToTensorV2())
 return process
def get_transforms(*, aug_list, cfg):
return Compose(
 _get_augmentations(aug_list, cfg)
 )
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))<load_from_csv>
class CustomModel(nn.Module):
def __init__(self, model_name, target_size, pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 if hasattr(self.model, 'classifier'):
n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, target_size)
 elif hasattr(self.model, 'fc'):
n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, target_size)
 elif hasattr(self.model, 'head'):
n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, target_size)
 def forward(self, x):
x = self.model(x)
 return x
test_dataset = reader.read('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 seq_iterator = BasicIterator(batch_size=64)
 seq_iterator.index_with(vocab )<save_to_csv>
def inference_tta(model, states, tta_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(tta_loader), total=len(tta_loader))
probs = []
for i,(images, _)in tk0:
images = images.to(device)
 batch_size, n_crops, c, h, w = images.size()
images = images.view(-1, c, h, w)
 avg_preds = []
for state in states:
model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(images ).softmax(1)
 y_preds = y_preds.view(batch_size, n_crops,-1)
 avg_preds.append(y_preds.to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 del images, _, y_preds, avg_preds
torch.cuda.empty_cache()
probs = np.concatenate(probs)
 return probs
predictor = ToxicCommentPredictor(model, seq_iterator, cuda_device=0)
 test_preds = predictor.predict(test_dataset)
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = test_preds
submission.to_csv('submission.csv', index=False )<import_modules>
def main_tta(config, model_dir):
INFO = config['info']
TAG = config['tag']
CFG = config['cfg']
CFG['train'] = False
CFG['inference'] = True
inference_batch_size = 8
seed_torch(seed=CFG['seed'])
 model = CustomModel(TAG['model_name'], CFG['target_size'], pretrained=False)
 states = [torch.load(path)for path in glob.glob(f'{model_dir}/*.pth')]
ttas = get_ttas(CFG)
 tta_dataset = TTADataset(test, TEST_PATH, ttas=ttas)
 tta_loader = DataLoader(tta_dataset, batch_size=inference_batch_size, shuffle=False,
num_workers=2, pin_memory=True)
 predictions = inference_tta(model, states, tta_loader, device)
 return predictions
import sys, os, re, csv, codecs, numpy as np, pandas as pd
import tensorflow as tf
from keras import backend as K
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
from keras.callbacks import Callback
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU, concatenate, MaxPooling1D
from keras.layers import Bidirectional, GlobalMaxPooling1D, Flatten, TimeDistributed, Conv1D, SpatialDropout1D, GlobalAveragePooling1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<define_variables>
data_num = len(test)
 model_num = len(model_dirs)
 target_num = CFG['target_size']
channel_num = 4
stage1_predictions = np.zeros(( model_num, data_num, channel_num, target_num), dtype=np.float)
 for config, model_dir in zip(tta_configs, tta_model_dirs):
stage1_predictions[model_dirs.index(model_dir)] = main_tta(config, model_dir)
 weights_opt_feats = stage1_predictions.mean(axis=2)
 stage1_predictions = stage1_predictions.transpose(1, 2, 0, 3 )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove840b300dtxt/glove.840B.300d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'<define_variables>
class StackingDataset(Dataset):
def __init__(self, X: np.ndarray, y: Optional[np.ndarray] = None):
self.X = X
self.y = y
def __len__(self):
return self.X.shape[0]
def __getitem__(self, idx):
if self.y is None:
return torch.tensor(self.X[idx], dtype=torch.float)
 else:
return(
 torch.tensor(self.X[idx], dtype=torch.float),
torch.tensor(self.y[idx], dtype=torch.long),
)
embed_size = 300
max_features = 150000
maxlen = 150<load_from_csv>
class CNNStacking(nn.Module):
def __init__(self, n_labels):
super(CNNStacking, self ).__init__()
self.sq = nn.Sequential(
 nn.Conv2d(in_channels=4, out_channels=8, kernel_size=(3, 1), bias=False),
nn.ReLU() ,
nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(3, 1), bias=False),
nn.ReLU() ,
nn.Flatten() ,
nn.Linear(in_features=16* n_labels, out_features=4 * n_labels),
nn.ReLU() ,
nn.Linear(in_features=4 * n_labels, out_features=n_labels), 
)
 def forward(self, x):
return self.sq(x )
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values 
<string_transform>
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(features)in tk0:
features = features.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(features)
 avg_preds.append(y_preds.softmax(1 ).to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen)
<categorify>
model = CNNStacking(CFG['target_size'])
 states = [torch.load(STAGE2_DIR+f'/fold{fold}_best.pth')for fold in CFG['trn_fold']]
test_dataset = StackingDataset(stage1_predictions)
 test_loader = DataLoader(test_dataset, batch_size=CFG['batch_size'], shuffle=False,
num_workers=CFG['num_workers'], pin_memory=True)
 pred_stacking = inference(model, states, test_loader, device)
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs<feature_engineering>
with open('.. /input/train-weights-optimization/best_weights.json', 'r')as f:
weights_dict = json.load(f)
 weights_dict
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
pred_weights_opt = np.zeros(weights_opt_feats.shape[1:], dtype=np.float)
 for idx, key in enumerate(model_dirs):
pred_weights_opt += weights_opt_feats[idx] * weights_dict[key[:-1]]
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 x = Bidirectional(LSTM(100, return_sequences=True,dropout=0.05,recurrent_dropout=0.05))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 adam = optimizers.Adam(lr=0.0015)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'] )<compute_train_metric>
BLENDING_WEIGHTS = {
"stacking": 0.5,
"weights_opt": 0.5
}
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))<split>
predictions = pred_stacking * BLENDING_WEIGHTS['stacking'] + pred_weights_opt * BLENDING_WEIGHTS['weights_opt']
predictions
<train_model><EOS> 
test['label'] = predictions.argmax(1)
 test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False )