Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<feature_engineering>
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )<define_variables>
from datetime import datetime
from glob import glob
from scipy.ndimage.interpolation import zoom
from skimage import io
from sklearn import metrics
from sklearn.metrics import log_loss
from sklearn.metrics import roc_auc_score, log_loss
from sklearn.model_selection import GroupKFold, StratifiedKFold
from torch import nn
from torch.cuda.amp import autocast, GradScaler
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torchvision import transforms
from tqdm import tqdm
import copy
import cv2
import joblib
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pydicom
import random
import sklearn
import time
import timm
import torch
import torchvision
import warnings
for i, e in enumerate(train['cast'][:1]):
print(i, e )<count_values>
CFG = {
'fold_num': 10,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 512,
'epochs': 32,
'train_bs': 28,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}
print('Number of casted persons in films')
 train['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(15 )<count_values>
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(15 )<count_unique_values>
train.label.value_counts()
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_genders for i in j] ).most_common()<count_values>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()
list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )<feature_engineering>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
img = get_img('.. /input/cassava-leaf-disease-classification/train_images/1000015157.jpg')
 plt.imshow(img)
 plt.show()
train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )<count_values>
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img
print('Number of casted persons in films')
 train['crew'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )<count_values>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion,
HueSaturationValue, IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur,
IAAPiecewiseAffine, RandomResizedCrop, IAASharpen, IAAEmboss,
RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,
ShiftScaleRotate, CenterCrop, Resize 
)
 def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0 
),
ToTensorV2(p=1.0)
 ], p=1.)
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )<count_values>
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15 )<count_values>
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds =(
 StratifiedKFold(n_splits=CFG['fold_num'])
.split(np.arange(train.shape[0]), train.label.values)
)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(
 valid_,
'.. /input/cassava-leaf-disease-classification/train_images/',
transforms=get_inference_transforms() ,
output_label=False 
)
 test = pd.DataFrame()
test['image_id'] = list(
 os.listdir('.. /input/cassava-leaf-disease-classification/test_images/')
)
 test_ds = CassavaDataset(
 test,
'.. /input/cassava-leaf-disease-classification/test_images/',
transforms=get_inference_transforms() ,
output_label=False 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(
 torch.load(
 '.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}' 
.format(CFG['model_arch'],fold, epoch)
)
)
 with torch.no_grad() :
for _ in range(CFG['tta']):
val_preds += [
CFG['weights'][i]
/sum(CFG['weights'])
 /CFG['tta']
*inference_one_epoch(model, val_loader, device)
 ]
tst_preds += [
CFG['weights'][i]
/sum(CFG['weights'])
 /CFG['tta']
*inference_one_epoch(model, tst_loader, device)
 ]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print('fold {} validation loss = {:.5f}'.format(
 fold,
log_loss(valid_.label.values, val_preds)
))
print('fold {} validation accuracy = {:.5f}'.format(
 fold, 
(valid_.label.values==np.argmax(val_preds, axis=1)).mean() 
))
del model
torch.cuda.empty_cache()
list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_genders for i in j] ).most_common(15 )<count_values>
df_test_predict_proba_1 = pd.concat(
 [test, pd.DataFrame(softmax(tst_preds, axis = 1)) ],
axis=1
).sort_values(["image_id"] )
list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_departments for i in j] ).most_common(14 )<feature_engineering>
variable_list = %who_ls
for _ in variable_list:
if _ is not "df_test_predict_proba_1":
del globals() [_]
%who_ls
train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
for g in top_crew_names:
train['crew_name_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['crew_character_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_crew_names:
test['crew_name_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['crew_character_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )<feature_engineering>
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassava-resnext50-32x4d-weights/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'
train['log_revenue'] = np.log1p(train['revenue'] )<feature_engineering>
class CFG:
debug=False
num_workers=8
model_name='resnext50_32x4d'
size=512
batch_size=32
seed=2020
target_size=5
target_col='label'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
inference=True
train['log_budget'] = np.log1p(train['budget'])
 test['log_budget'] = np.log1p(test['budget'] )<count_values>
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 warnings.filterwarnings('ignore' )
train['homepage'].value_counts().head()<import_modules>
def get_score(y_true, y_pred):
return accuracy_score(y_true, y_pred)
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def init_logger(log_file=OUTPUT_DIR+'inference.log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed )
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model import LinearRegression
import eli5<train_model>
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test.head()
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = vectorizer.fit_transform(train['overview'].fillna(''))
linreg = LinearRegression()
linreg.fit(overview_text, train['log_revenue'])
 eli5.show_weights(linreg, vec=vectorizer, top=20, feature_filter=lambda x: x != '<BIAS>' )<feature_engineering>
def get_transforms(*, data):
if data == 'valid':
return A.Compose([
A.Resize(CFG.size, CFG.size),
A.Transpose(p=0.5),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.ShiftScaleRotate(p=0.5),
A.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2()
] )
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98'<categorify>
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year<data_type_conversions>
def load_state(model_path):
model = CustomResNext(CFG.model_name, pretrained=False)
 try:
model.load_state_dict(torch.load(model_path)['model'], strict=True)
 state_dict = torch.load(model_path)['model']
except:
state_dict = torch.load(model_path)['model']
state_dict = {
k[7:]
if k.startswith('module.')
 else k: state_dict[k]
for k in state_dict.keys()
}
return state_dict
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state)
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.softmax(1 ).to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 test['release_date'] = pd.to_datetime(test['release_date'] )<data_type_conversions>
model = CustomResNext(CFG.model_name, pretrained=False)
 states = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(
 test_dataset,
batch_size=CFG.batch_size,
shuffle=False,
num_workers=CFG.num_workers,
pin_memory=True 
)
 predictions = inference(model, states, test_loader, device )
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train = process_date(train)
 test = process_date(test )<set_options>
df_test_predict_proba_2 = pd.concat(
 [test["image_id"], pd.DataFrame(softmax(predictions, axis = 1)) ],
axis=1
).sort_values(["image_id"] )
py.init_notebook_mode(connected=True)
<count_values>
submission = df_test_predict_proba_1[["image_id"]]
submission["label"] =(
 df_test_predict_proba_1.drop(["image_id"], axis=1)* 0.5
+ df_test_predict_proba_2.drop(["image_id"], axis=1)* 0.5
).to_numpy().argmax(1)
train['status'].value_counts()<count_values>
submission.to_csv("submission.csv", index=False )
<drop_column><EOS> 
submission.to_csv("submission.csv", index=False )