Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<drop_column>
batch_size = 1
image_size = 512
enet_type = ['tf_efficientnet_b4_ns'] * 5
model_path = ['.. /input/moa-b4-baseline/baseline_cld_fold0_epoch8_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold1_epoch9_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold2_epoch9_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold3_epoch5_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold4_epoch11_tf_efficientnet_b4_ns_512.pth']
to_drop_notV = to_drop[:14]
to_drop_V = to_drop[14:]
for col in to_drop_notV:
na_proportion = train[col].isna().sum() /train.shape[0]
if na_proportion >= 0.4:
train = train.drop(col,axis=1)
 test = test.drop(col,axis=1)
 print('Column {} was dropped'.format(col))
train = train.drop(columns = to_drop_V)
 test = test.drop(columns = to_drop_V)
 print('
{} V columns were dropped'.format(len(to_drop_V)))
<data_type_conversions>
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 DEBUG = False
%matplotlib inline
device = torch.device('cuda')if not DEBUG else torch.device('cpu' )
num_cols = list(set(num_cols)- set(['D2','D6','D7'])- set(to_drop_V))
train[cat_cols] = train[cat_cols].astype('str' ).replace('nan','unknown' ).astype('category')
 test[cat_cols] = test[cat_cols].astype('str' ).replace('nan','unknown' ).astype('category')
 train[num_cols] = train[num_cols].fillna(-999)
 test[num_cols] = test[num_cols].fillna(-999 )<count_unique_values>
transforms_valid = albumentations.Compose([
albumentations.CenterCrop(image_size, image_size, p=1),
albumentations.Resize(image_size, image_size),
albumentations.Normalize()
] )
print('Showing the first 10 unique elements of DeviceInfo(out of {}):
{}'\ 
.format(len(list(train['DeviceInfo'].unique())) ,\
list(train['DeviceInfo'].unique())[:10]))<feature_engineering>
class CLDDataset(Dataset):
def __init__(self, df, mode, transform=None):
self.df = df.reset_index(drop=True)
 self.mode = mode
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
row = self.df.loc[index]
image = cv2.imread(row.filepath)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform is not None:
res = self.transform(image=image)
 image = res['image']
image = image.astype(np.float32)
 image = image.transpose(2,0,1)
 if self.mode == 'test':
return torch.tensor(image ).float()
else:
return torch.tensor(image ).float() , torch.tensor(row.label ).float()
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test['filepath'] = test.image_id.apply(lambda x: os.path.join('.. /input/cassava-leaf-disease-classification/test_images', f'{x}'))
test_dataset = CLDDataset(test, 'test', transform=transforms_valid)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4 )
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('sm', na=False), 'device_name'] = 'samsung'
df.loc[df['device_name'].str.contains('samsung', na=False), 'device_name'] = 'samsung'
df.loc[df['device_name'].str.contains('gt-', na=False), 'device_name'] = 'samsung'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'motorola'
df.loc[df['device_name'].str.contains('lg', na=False), 'device_name'] = 'lg'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'rv'
df.loc[df['device_name'].str.contains('huawei', na=False), 'device_name'] = 'huawei'
df.loc[df['device_name'].str.contains('ale-', na=False), 'device_name'] = 'huawei'
df.loc[df['device_name'].str.contains('-l', na=False), 'device_name'] = 'huawei'
df.loc[df['device_name'].str.contains('blade', na=False), 'device_name'] = 'zte'
df.loc[df['device_name'].str.contains('linux', na=False), 'device_name'] = 'linux'
df.loc[df['device_name'].str.contains('xt', na=False), 'device_name'] = 'sony'
df.loc[df['device_name'].str.contains('htc', na=False), 'device_name'] = 'htc'
df.loc[df['device_name'].str.contains('asus', na=False), 'device_name'] = 'asus'
df.loc[df['device_name'].str.contains('lenovo', na=False), 'device_name'] = 'lenovo'
df.loc[df['device_name'].str.contains('pixel', na=False), 'device_name'] = 'google'
df.loc[df['device_name'].str.contains('redmi', na=False), 'device_name'] = 'xiaomi'
df.loc[df['device_name'].str.contains('windows', na=False), 'device_name'] = 'microsoft'
df.loc[df['device_name'].str.contains('microsoft', na=False), 'device_name'] = 'microsoft'
df.loc[df['device_name'].str.contains('nexus', na=False), 'device_name'] = 'google'
df.loc[df['device_name'].str.contains('ilium', na=False), 'device_name'] = 'lanix'
df.loc[df['device_name'].str.contains('lumia', na=False), 'device_name'] = 'nokia'
df['DeviceInfo'] = df['DeviceInfo'].astype('category')
 df['device_name'] = df['device_name'].astype('category')
 return df<data_type_conversions>
class enet_v2(nn.Module):
def __init__(self, backbone, out_dim, pretrained=False):
super(enet_v2, self ).__init__()
self.enet = timm.create_model(backbone, pretrained=pretrained)
 in_ch = self.enet.classifier.in_features
self.myfc = nn.Linear(in_ch, out_dim)
 self.enet.classifier = nn.Identity()
def forward(self, x):
x = self.enet(x)
 x = self.myfc(x)
 return x
train['id_31'] = train['id_31'].str.lower().astype('category')
 test['id_31'] = test['id_31'].str.lower().astype('category' )<data_type_conversions>
test_preds = []
for i in range(len(enet_type)) :
model = enet_v2(enet_type[i], out_dim=5)
 model = model.to(device)
 model.load_state_dict(torch.load(model_path[i]))
test_preds += [tta_inference_func(test_loader)]
train['id_30'] = train['id_30'].str.lower()
test['id_30'] = test['id_30'].str.lower()
train['id_30_word1'] = train['id_30'].str.split(' ', expand=True)[0].astype('category')
 test['id_30_word1'] = test['id_30'].str.split(' ', expand=True)[0].astype('category')
 train['id_30'] = train['id_30'].astype('category')
 test['id_30'] = test['id_30'].astype('category')
 print('The new column, id_30_word1, has {} unique values'\ 
.format(len(list(train['id_30_word1'].unique()))))<data_type_conversions>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.label = np.argmax(np.mean(test_preds, axis=0), axis=1)
 submission.to_csv('submission.csv', index=False )