Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<save_to_csv>
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
sys.path.append(package_path)
 DATA_DIR = '.. /input/cassava-leaf-disease-classification'
MODEL_DIR_0 = '.. /input/gpu-vit-noisearch-amp-aug-fold-0'
MODEL_DIR_1 = '.. /input/gpu-vit-noisearch-amp-aug-fold-1'
MODEL_DIR_2 = '.. /input/gpu-vit-noisearch-amp-aug-fold-2'
MODEL_DIR_3 = '.. /input/gpu-vit-noisearch-amp-aug-fold-3'
MODEL_DIR_4 = '.. /input/gpu-vit-noisearch-amp-aug-fold-4'
MODEL_DIR_01 = '.. /input/gpu-en-b4-ns-noisearch-amp-aug-fold-0'
MODEL_DIR_11 = '.. /input/gpu-en-b4-ns-noisearch-amp-aug-fold-1'
MODEL_DIR_21 = '.. /input/gpu-en-b4-ns-noisearch-amp-aug-fold-2'
MODEL_DIR_31 = '.. /input/gpu-en-b4-ns-noisearch-amp-aug-fold-3'
MODEL_DIR_41 = '.. /input/gpu-en-b4-ns-noisearch-amp-aug-fold-4'
MODEL_DIR_02 = '.. /input/gpu-seresnext-noisearch-amp-aug-fold-0'
MODEL_DIR_12 = '.. /input/gpu-seresnext-noisearch-amp-aug-fold-1'
MODEL_DIR_22 = '.. /input/gpu-seresnext-noisearch-amp-aug-fold-2'
MODEL_DIR_32 = '.. /input/gpu-seresnext-noisearch-amp-aug-fold-3'
MODEL_DIR_42 = '.. /input/gpu-seresnext-noisearch-amp-aug-fold-4'
MODEL_DIR_03 = '.. /input/gpu-en-b4-ns-amp-aug-fold-0'
MODEL_DIR_13 = '.. /input/gpu-en-b4-ns-amp-aug-fold-1'
MODEL_DIR_23 = '.. /input/gpu-en-b4-ns-amp-aug-fold-2'
MODEL_DIR_33 = '.. /input/gpu-en-b4-ns-amp-aug-fold-3'
MODEL_DIR_43 = '.. /input/gpu-en-b4-ns-amp-aug-fold-4'
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('submission.csv', index=False )<load_from_csv>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )<count_values>
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'vit_base_patch16_384',
'img_size': 384,
'epochs': 10,
'train_bs': 16,
'valid_bs': 16,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [7,8,9],
'weights': [1,1,1,1,1,1]
}
value_counts = train_data.iloc[:,2:].apply(pd.Series.value_counts)
 print(value_counts.iloc[1])
<choose_model_class>
EPOCHS0 = {
0: [9,8,6,5],
1: [5,6,4,8],
2: [8,9,7,6],
3: [8,7,9,6],
4: [9,8,7,4]
}
EPOCHS1 = {
0: [8,9,7,6],
1: [9,4,8,6],
2: [9,7,8,4],
3: [5,8,9,3],
4: [6,7,8,9]
}
EPOCHS2 = {
0: [8,9,6,7],
1: [9,8,5,6],
2: [9,7,5,4],
3: [5,8,9,4],
4: [5,8,9,7]
}
EPOCHS3 = {
0: [8,9,7,6],
1: [2,7,9,5],
2: [5,6,9,7],
3: [8,9,7,6],
4: [4,5,8,7]
}
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
max_features=30000)
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 4),
max_features=30000)
<concatenate>
train = pd.read_csv(f'{DATA_DIR}/train.csv' )
vectorizer = make_union(word_vectorizer, char_vectorizer, n_jobs=3 )<feature_engineering>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
train_comments = train_data['comment_text']
test_comments = test_data['comment_text']
vectorizer.fit(train_comments)
 train_features = vectorizer.transform(train_comments)
 test_features = vectorizer.transform(test_comments )<compute_train_metric>
class CassavaDataset(Dataset):
def __init__(self, df, data_root, transforms=None, output_label=True):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
img = get_img(f"{self.data_root}/{self.df.loc[index]['image_id']}")
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label:
return img, target
else:
return img
scores = []
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
submission = pd.DataFrame.from_dict({'id': test_data['id']})
 for class_name in class_names:
train_target = train_data[class_name]
classifier = LogisticRegression(solver='sag')
 cv_score = np.mean(cross_val_score(
 classifier, train_features, train_target, cv=3, scoring='roc_auc'))
scores.append(cv_score)
 print('CV score for class {} is {}'.format(class_name, cv_score))
classifier.fit(train_features, train_target)
 submission[class_name] = classifier.predict_proba(test_features)[:, 1]
print('Total CV score is {}'.format(np.mean(scores)))
 submission.to_csv('submission.csv', index=False )<import_modules>
class CassvaImgClassifierN(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
from datetime import date, timedelta
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
import lightgbm as lgb<load_from_csv>
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.n_class = n_class
self.model = timm.create_model(model_arch, pretrained=pretrained)
 if 'vit' in model_arch:
n_features = self.model.head.in_features
self.model.head = nn.Identity()
if 'eff' in model_arch:
n_features = self.model.classifier.in_features
self.model.classifier = nn.Identity()
if 'resnext' in model_arch:
n_features = self.model.fc.in_features
self.model.fc = nn.Identity()
self.layer_noise = nn.Linear(n_features, n_class)
 self.layer_pure = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 x1 = self.layer_noise(x)
 x2 = self.layer_pure(x)
 x1 = x1 + x2
return x1, x2
df_train = pd.read_csv(
 '.. /input/train.csv', usecols=[1, 2, 3, 4, 5],
dtype={'onpromotion': bool},
converters={'unit_sales': lambda u: np.log1p(
 float(u)) if float(u)> 0 else 0},
parse_dates=["date"],
skiprows=range(1, 66458909)
)
 df_test = pd.read_csv(
 ".. /input/test.csv", usecols=[0, 1, 2, 3, 4],
dtype={'onpromotion': bool},
parse_dates=["date"]
).set_index(
 ['store_nbr', 'item_nbr', 'date'] 
)
 items = pd.read_csv(
 ".. /input/items.csv",
).set_index("item_nbr" )<drop_column>
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
pbar = tqdm(enumerate(data_loader), total=len(data_loader))
for step,(imgs)in pbar:
imgs = imgs.to(device ).float()
_, image_preds = model(imgs)
 image_preds_all += [torch.sigmoid(image_preds ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all
df_2017 = df_train[df_train.date.isin(
 pd.date_range("2017-05-31", periods=7 * 11)) ].copy()
del df_train<concatenate>
def run_inference(fold, MODEL_DIR, model_arch, epoch_list):
print(f'Inference fold {fold} started')
 test = pd.DataFrame()
test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))
if 'vit' in model_arch:
T = get_inference_transforms_384()
else:
T = get_inference_transforms()
test_ds = CassavaDataset(
 test, f'{DATA_DIR}/test_images/',
transforms=T, output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(model_arch, train.label.nunique() ).to(device)
 tst_preds = []
for i, epoch in enumerate(epoch_list[fold][:1]):
model.load_state_dict(torch.load(f'{MODEL_DIR}/{model_arch}_fold_{fold}_{epoch}'))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_image_preds = inference_one_epoch(model, tst_loader, device)
 tst_preds += [(1/CFG['tta'])*tst_image_preds]
tst_preds = np.sum(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()
return tst_preds
promo_2017_train = df_2017.set_index(
 ["store_nbr", "item_nbr", "date"])[["onpromotion"]].unstack(
 level=-1 ).fillna(False)
 promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)
 promo_2017_test = df_test[["onpromotion"]].unstack(level=-1 ).fillna(False)
 promo_2017_test.columns = promo_2017_test.columns.get_level_values(1)
 promo_2017_test = promo_2017_test.reindex(promo_2017_train.index ).fillna(False)
 promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)
 del promo_2017_test, promo_2017_train<sort_values>
def run_inferenceN(fold, MODEL_DIR, model_arch):
print(f'Inference fold {fold} started')
 test = pd.DataFrame()
test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))
test_ds = CassavaDataset(
 test, f'{DATA_DIR}/test_images/',
transforms=get_inference_transforms() , output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifierN(model_arch, train.label.nunique() ).to(device)
 tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(
 torch.load(f'{MODEL_DIR}/{model_arch}_fold_{fold}_{epoch}'))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_image_preds = inference_one_epochN(model, tst_loader, device)
 tst_preds += [(1/ CFG['tta'])*tst_image_preds]
tst_preds = np.mean(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()
return tst_preds
items = items.reindex(df_2017.index.get_level_values(1))
items.head()<feature_engineering>
preds0 = run_inference(0, MODEL_DIR_0, 'vit_base_patch16_384', EPOCHS0)
 preds1 = run_inference(1, MODEL_DIR_1, 'vit_base_patch16_384', EPOCHS0)
 preds2 = run_inference(2, MODEL_DIR_2, 'vit_base_patch16_384', EPOCHS0)
 preds3 = run_inference(3, MODEL_DIR_3, 'vit_base_patch16_384', EPOCHS0)
 preds4 = run_inference(4, MODEL_DIR_4, 'vit_base_patch16_384', EPOCHS0)
 PRED0 =(preds0 + preds1 + preds2 + preds3 + preds4)/5
def get_timespan(df, dt, minus, periods):
return df[
pd.date_range(dt - timedelta(days=minus), periods=periods)
 ]<create_dataframe>
preds0 = run_inference(0, MODEL_DIR_01, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds1 = run_inference(1, MODEL_DIR_11, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds2 = run_inference(2, MODEL_DIR_21, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds3 = run_inference(3, MODEL_DIR_31, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds4 = run_inference(4, MODEL_DIR_41, 'tf_efficientnet_b4_ns', EPOCHS1)
 PRED1 =(preds0 + preds1 + preds2 + preds3 + preds4)/5
def prepare_dataset(t2017, is_train=True):
X = pd.DataFrame({
"mean_3_2017": get_timespan(df_2017, t2017, 3, 3 ).mean(axis=1 ).values,
"mean_7_2017": get_timespan(df_2017, t2017, 7, 7 ).mean(axis=1 ).values,
"mean_14_2017": get_timespan(df_2017, t2017, 14, 14 ).mean(axis=1 ).values,
"promo_14_2017": get_timespan(promo_2017, t2017, 14, 14 ).sum(axis=1 ).values
})
 for i in range(16):
X["promo_{}".format(i)] = promo_2017[
t2017 + timedelta(days=i)].values.astype(np.uint8)
 if is_train:
y = df_2017[
pd.date_range(t2017, periods=16)
 ].values
return X, y
return X<prepare_x_and_y>
preds0 = run_inference(0, MODEL_DIR_02, 'seresnext50_32x4d', EPOCHS2)
 preds1 = run_inference(1, MODEL_DIR_12, 'seresnext50_32x4d', EPOCHS2)
 preds2 = run_inference(2, MODEL_DIR_22, 'seresnext50_32x4d', EPOCHS2)
 preds3 = run_inference(3, MODEL_DIR_32, 'seresnext50_32x4d', EPOCHS2)
 preds4 = run_inference(4, MODEL_DIR_42, 'seresnext50_32x4d', EPOCHS2)
 PRED2 =(preds0 + preds1 + preds2 + preds3 + preds4)/5
print("Preparing dataset...")
 t2017 = date(2017, 6, 21)
 X_l, y_l = [], []
for i in range(6):
delta = timedelta(days=7 * i)
 X_tmp, y_tmp = prepare_dataset(
 t2017 + delta 
)
 X_l.append(X_tmp)
 y_l.append(y_tmp)
 X_train = pd.concat(X_l, axis=0)
 y_train = np.concatenate(y_l, axis=0)
 del X_l, y_l
X_test = prepare_dataset(date(2017, 8, 16), is_train=False )<train_model>
preds0 = run_inferenceN(0, MODEL_DIR_03, 'tf_efficientnet_b4_ns',)
 preds1 = run_inferenceN(1, MODEL_DIR_13, 'tf_efficientnet_b4_ns',)
 preds2 = run_inferenceN(2, MODEL_DIR_23, 'tf_efficientnet_b4_ns',)
 preds3 = run_inferenceN(3, MODEL_DIR_33, 'tf_efficientnet_b4_ns',)
 preds4 = run_inferenceN(4, MODEL_DIR_43, 'tf_efficientnet_b4_ns')
 PRED3 =(preds0 + preds1 + preds2 + preds3 + preds4)/5
print("Training and predicting models...")
 params = {
'num_leaves': 2**5 - 1,
'objective': 'regression_l2',
'max_depth': 8,
'min_data_in_leaf': 50,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 1,
'metric': 'l2',
'num_threads': 4
}<define_variables>
tst_preds =(PRED0 + 2*PRED1 + PRED2 + PRED3)/5
MAX_ROUNDS = 1000
val_pred = []
test_pred = []
cate_vars = []
for i in range(16):
print("=" * 50)
 print("Step %d" %(i+1))
print("=" * 50)
 dtrain = lgb.Dataset(
 X_train, label=y_train[:, i],
categorical_feature=cate_vars,
weight=pd.concat([items["perishable"]] * 6)* 0.25 + 1 
)
 bst = lgb.train(
 params, dtrain, num_boost_round=700,
valid_sets=[dtrain], verbose_eval=50 
)
 print("
".join(( "%s: %.2f" % x)for x in sorted(
 zip(X_train.columns, bst.feature_importance("gain")) ,
key=lambda x: x[1], reverse=True 
)))
 test_pred.append(bst.predict(
 X_test, num_iteration=700))<compute_test_metric>
test = pd.DataFrame()
test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))
test['label'] = np.argmax(tst_preds, axis=1)
 test.to_csv('submission.csv', index=False )