Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<filter>
tez_path = '.. /input/tez-lib'
effnet_path = '.. /input/efficientnet-pytorch/'
sys.path.append(tez_path)
 sys.path.append(effnet_path )
print(train[train['length_clean_review'] == 4]['review'].iloc[0])
 print('------After Cleaning------')
 print(train[train['length_clean_review'] == 4]['clean_review'].iloc[0] )<import_modules>
import os
import albumentations
import pandas as pd
import numpy as np
import tez
from tez.datasets import ImageDataset
import torch
import torch.nn as nn
from torch.nn import functional as F
from efficientnet_pytorch import EfficientNet
from tqdm import tqdm
from sklearn.feature_extraction.text import CountVectorizer<feature_engineering>
class LeafModel(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.effnet = EfficientNet.from_name("efficientnet-b4")
 self.dropout = nn.Dropout(0.2)
 self.out = nn.Linear(1792, num_classes)
 self.step_scheduler_after = "epoch"
def forward(self, image, targets=None):
batch_size, _, _, _ = image.shape
x = self.effnet.extract_features(image)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 outputs = self.out(self.dropout(x))
return outputs, None, None
def load(self, model_path, device="cuda"):
self.device = device
if next(self.parameters() ).device != self.device:
self.to(self.device)
 model_dict = torch.load(model_path, map_location=device)
 self.load_state_dict(model_dict["state_dict"])
 def predict(self, dataset, sampler=None, batch_size=16, n_jobs=1, collate_fn=None):
if next(self.parameters() ).device != self.device:
self.to(self.device)
 if batch_size == 1:
n_jobs = 0
data_loader = torch.utils.data.DataLoader(
 dataset, batch_size=batch_size, num_workers=4, sampler=sampler, collate_fn=collate_fn, pin_memory=True 
)
 if self.training:
self.eval()
final_output = []
tk0 = tqdm(data_loader, total=len(data_loader))
for b_idx, data in enumerate(tk0):
with torch.no_grad() :
out = self.predict_one_step(data)
 out = out.cpu().detach().numpy()
yield out
tk0.set_postfix(stage="test")
 tk0.close()
def model_fn(self, data):
for key, value in data.items() :
data[key] = value.to(self.device)
 output, loss, metrics = self(**data)
 return output, loss, metrics
def predict_one_step(self, data):
output, _, _ = self.model_fn(data)
 return output
bow_transform = CountVectorizer(analyzer=clean_text ).fit(train['review'])
 print(len(bow_transform.vocabulary_))<categorify>
test_aug = albumentations.Compose([
albumentations.RandomResizedCrop(512, 512),
albumentations.Transpose(p=0.5),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
albumentations.RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
albumentations.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0 
)
 ], p=1.)
review_bow = bow_transform.transform(train['review'] )<train_model>
dfx = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 image_path = ".. /input/cassava-leaf-disease-classification/test_images/"
test_image_paths = [os.path.join(image_path, x)for x in dfx.image_id.values]
test_targets = dfx.label.values
test_dataset = ImageDataset(
 image_paths=test_image_paths,
targets=test_targets,
augmentations=test_aug,
)
sparsity =(100.0 * review_bow.nnz /(review_bow.shape[0] * review_bow.shape[1]))
print('sparsity: {}'.format(sparsity))<categorify>
train_dfx = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 model0 = LeafModel(num_classes=train_dfx.label.nunique())
 model0.load(".. /input/single-model/efficentnet_model_fold0.bin", device='cuda' )
tfidf_transformer = TfidfTransformer().fit(review_bow)
 tfidf1 = tfidf_transformer.transform(bow1)
 print(tfidf1 )<feature_engineering>
model_list = [model0]
def run_inference(model):
final_preds = None
for j in range(20):
preds = model.predict(test_dataset, batch_size=64, n_jobs=-1)
 temp_preds = None
for p in preds:
if temp_preds is None:
temp_preds = p
else:
temp_preds = np.vstack(( temp_preds, p))
if final_preds is None:
final_preds = temp_preds
else:
final_preds += temp_preds
final_preds /= 20
final_preds = final_preds.argmax(axis=1)
 return final_preds
<categorify><EOS> 
dfx.label = run_inference(model_list[0])
 dfx.to_csv("submission.csv", index=False )