Kaggle competition: Cassava Leaf Disease Classification
train['has_collection'] = train['belongs_to_collection'].apply(lambda x: 1 if str(x)!= 'nan' else 0)
 train['collection_id'] = train['belongs_to_collection'].apply(lambda x: eval(x)[0]['id'] if str(x)!= 'nan' else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: 1 if str(x)!= 'nan' else 0)
 test['collection_id'] = test['belongs_to_collection'].apply(lambda x: eval(x)[0]['id'] if str(x)!= 'nan' else 0 )<drop_column>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )<define_variables>
CFG = {
'img_size': 512,
'tta': 3,
'valid_bs': 16,
'device': 'cuda' if torch.cuda.is_available() else 'cpu',
'effnet_models': ['model_4.pt', 'model_5.pt', 'model_6.pt', 'model_7.pt'],
'resnet_models': ['model_6.pt', 'model_8.pt', 'model_9.pt', 'model_10.pt']
}
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in eval(x)] if str(x)!= 'nan' else [] ).values )<feature_engineering>
class DiseaseDatasetInference(torch.utils.data.Dataset):
def __init__(self, df, transform=None, opt_label=True):
self.df = df.reset_index(drop=True ).copy()
self.transform = transform
self.opt_label = opt_label
if self.opt_label:
self.data = [(row['image_id'], row['label'])for _, row in self.df.iterrows() ]
else:
self.data = [(row['image_id'])for _, row in self.df.iterrows() ]
self.data = np.asarray(self.data)
 def __len__(self):
return len(self.data)
 def __getitem__(self, index):
if self.opt_label:
image_path, label = self.data[index]
else:
image_path = self.data[index]
image = cv2.imread(image_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform is not None:
image = self.transform(image=image)['image']
if self.opt_label == True:
return(image, int(label))
else:
return image
top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in eval(x)])) if(isinstance(x,int)or isinstance(x,str)) == True else '')
 for gen in top_genres:
train['genre_' + gen] = train['all_genres'].apply(lambda x: 1 if gen in x else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in eval(x)])) if(isinstance(x,int)or isinstance(x,str)) == True else '')
 for gen in top_genres:
test['genre_' + gen] = test['all_genres'].apply(lambda x: 1 if gen in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )<categorify>
df = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/sample_submission.csv')
 PATH = '/kaggle/input/cassava-leaf-disease-classification/test_images/'
lang_encoder = LabelEncoder()
train['all_genres'] = lang_encoder.fit_transform(train['all_genres'])
 test['all_genres'] = lang_encoder.fit_transform(test['all_genres'] )<categorify>
test_csv = df.copy()
test_csv['image_id'] = PATH + test_csv['image_id']
test_ds = DiseaseDatasetInference(test_csv, transform=get_inference_transforms() , opt_label=False)
 test_loader = torch.utils.data.DataLoader(test_ds, batch_size=CFG['valid_bs'], shuffle=False, pin_memory=False )
lang_encoder = LabelEncoder()
train['original_language'] = lang_encoder.fit_transform(train['original_language'])
 test['original_language'] = lang_encoder.fit_transform(test['original_language'] )<define_variables>
def inference(model, data_loader, device):
preds = []
model.eval()
test_tqdm = tq.tqdm(data_loader, total=len(data_loader), desc="Testing", position=0, leave=True)
 for images in test_tqdm:
images = images.to(device)
 preds.extend(model(images ).detach().cpu().numpy())
 return preds
prod_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in eval(x)] if str(x)!= 'nan' else '' ).values )<feature_engineering>
class CassavaImageClassifier(nn.Module):
def __init__(self, efnet_arch, n_class, pretrained=False):
super().__init__()
self.efnet_model = timm.create_model(efnet_arch, pretrained=pretrained)
 efnet_features = self.efnet_model.classifier.in_features
self.efnet_model.classifier = nn.Linear(efnet_features, n_class)
 def forward(self, x):
efnet_opt = self.efnet_model(x)
 return efnet_opt
train['prod_companies_count'] = train['production_companies'].apply(lambda x: len([i for i in eval(x)])if str(x)!= 'nan' else 0)
 test['prod_companies_count'] = test['production_companies'].apply(lambda x: len([i for i in eval(x)])if str(x)!= 'nan' else 0 )<define_variables>
effnet_preds = []
for effnet_model_name in CFG['effnet_models']:
print("Model: ", effnet_model_name)
 effnet_model = torch.load('/kaggle/input/effete-cassava/ensemble/'+effnet_model_name, map_location=torch.device(CFG['device']))
with torch.no_grad() :
for i in range(CFG['tta']):
effnet_preds += [inference(effnet_model, test_loader, CFG['device'])]
effnet_preds = np.mean(effnet_preds, axis=0 )
pop_production = Counter([i for j in prod_companies for i in j] )<feature_engineering>
effnet_outcomes = pd.concat([df['image_id'], pd.DataFrame(effnet_preds)], axis=1 ).sort_values(['image_id'] )
train['production_score'] = train['production_companies'].apply(lambda x: np.tanh(max([pop_production[i['name']] for i in eval(x)])) if str(x)!= 'nan' else 0)
 test['production_score'] = test['production_companies'].apply(lambda x: np.tanh(max([pop_production[i['name']] for i in eval(x)])) if str(x)!= 'nan' else 0 )<drop_column>
class CassavaImageClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
train = train.drop(['production_companies'], axis=1)
 test = test.drop(['production_companies'], axis=1 )<feature_engineering>
resnet_preds = []
for resnet_model_name in CFG['resnet_models']:
print("Model: ", resnet_model_name)
 resnet_model = torch.load('/kaggle/input/resnet-cassava/resnet/'+resnet_model_name, map_location=torch.device(CFG['device']))
with torch.no_grad() :
for i in range(CFG['tta']):
resnet_preds += [inference(resnet_model, test_loader, CFG['device'])]
resnet_preds = np.mean(resnet_preds, axis=0 )
train['production_countries'] = train['production_countries'].apply(lambda x: [i['name'] for i in eval(x)][0] if str(x)!= 'nan' else '')
 test['production_countries'] = test['production_countries'].apply(lambda x: [i['name'] for i in eval(x)][0] if str(x)!= 'nan' else '' )<categorify>
resnet_outcomes = pd.concat([df['image_id'], pd.DataFrame(resnet_preds)], axis=1 ).sort_values(['image_id'] )
prod_country_encoder = LabelEncoder()
train['production_countries'] = prod_country_encoder.fit_transform(train['production_countries'])
 test['production_countries'] = prod_country_encoder.fit_transform(test['production_countries'] )<data_type_conversions>
final_preds =(effnet_outcomes.drop('image_id', axis=1)*0.5 + resnet_outcomes.drop('image_id', axis=1)*0.5 ).to_numpy()
final_preds = softmax(final_preds ).argmax(1 )
train['release_date'] = train['release_date'].apply(lambda x: pd.to_datetime(x))
test['release_date'] = test['release_date'].apply(lambda x: pd.to_datetime(x))<feature_engineering>
accuracy_score(final_preds, df['label'].values )
<drop_column><EOS> 
submit = pd.DataFrame({'image_id': df['image_id'].values, 'label': final_preds})
 submit.to_csv('submission.csv', index=False )