Kaggle competition: Cassava Leaf Disease Classification
dd=count_rev['production_countries'].value_counts()
mn['values']=dd.values
mn.head()<categorify>
package_path = '.. /input/vision-transformer-pytorch/VisionTransformer-Pytorch'
sys.path.append(package_path )
mn['address'][0]='United States'
mn['address'][2]='Korea'
mn['address'][1]='India'
mn['address'][3]='Serbia'
mn['address'][4]='United Kingdom'
mn['address'][5]='France'
mn['address'][6]='New Zealand'
mn['address'][7]='Italy'
mn['address'][8]='Belgium'
mn['address'][9]='Czech Rep.'
mn['address'][11]='Russia'
mn['address'][12]='Spain'
mn['address'][13]='Turkey'
mn['address'][14]='China'
mn['address'][15]='Canada'
mn['address'][16]='Australia'
mn['address'][17]='Iran'
mn['address'][18]='Japan'
mn['address'][19]='Sweden'
mn['address'][20]='Philippines'
mn['address'][21]='Brazil'
mn['address'][22]='Netherlands'
mn['address'][23]='Ireland'
mn['address'][24]='Mexico'
mn['address'][25]='Germany'
mn['address'][26]='South Africa'
mn['address'][27]='Finland'
mn['address'][28]='Pakistan'
mn['address'][29]='Norway'
mn['address'][30]='Bulgaria'
mn['address'][31]='Ukraine'
mn['address'][32]='Denmark'
mn['address'][33]='Romania'
mn['address'][35]='Hungary'
mn['address'][36]='Chile'
mn['address'][37]='Indonesia'
mn['address'][38]='Poland' 
<drop_column>
import os
import pandas as pd
import albumentations as albu
import matplotlib.pyplot as plt
import json
import seaborn as sns
import cv2
import albumentations as albu
import numpy as np
mn=mn.drop(10)
 mn=mn.drop(34 )<feature_engineering>
BASE_DIR=".. /input/cassava-leaf-disease-classification/"
TRAIN_IMAGES_DIR=os.path.join(BASE_DIR,'train_images')
 train_df=pd.read_csv(os.path.join(BASE_DIR,'train.csv'))
mn['geometry']='unknow'<feature_engineering>
print("Count of training images {0}".format(len(os.listdir(TRAIN_IMAGES_DIR))))
for i in range(39):
if i==10 or i==34:
continue;
d=world[world['name']==mn['address'][i]]['geometry'].values[0]
mn['geometry'][i]=d<feature_engineering>
with open(f'{BASE_DIR}/label_num_to_disease_map.json', 'r')as f:
name_mapping = json.load(f)
 name_mapping = {int(k): v for k, v in name_mapping.items() }
train_df["class_id"]=train_df["label"].map(name_mapping )
TS = train.loc[:,["original_title","release_date","budget","runtime","revenue"]]
TS.dropna()
TS.release_date = pd.to_datetime(TS.release_date)
 TS.loc[:,"Year"] = TS["release_date"].dt.year
TS.loc[:,"Month"] = TS["release_date"].dt.month
TS = TS[TS.Year<2018]<sort_values>
import torch
import torch.nn as nn
import torchvision.models as models
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from torch.optim.lr_scheduler import ReduceLROnPlateau
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split
from albumentations.pytorch import ToTensorV2
import time
import datetime
import copy
top3 = train.sort_values(by='popularity',ascending=False)[:10]
id3=top3[['title','poster_path','revenue']]<count_values>
class CassavaDataset(Dataset):
def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):
self.df=df
self.imfolder=imfolder
self.train=train
self.transforms=transforms
def __getitem__(self,index):
im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])
 x=cv2.imread(im_path,cv2.IMREAD_COLOR)
 x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)
 if(self.transforms):
x=self.transforms(image=x)['image']
if(self.train):
y=self.df.iloc[index]['label']
return x,y
else:
return x
def __len__(self):
return len(self.df )
old_title=new_data['title'].value_counts()<prepare_x_and_y>
train, valid = train_test_split(
 train_df,
test_size=0.1,
random_state=42,
stratify=train_df.label.values 
)
 train = train.reset_index(drop=True)
 valid = valid.reset_index(drop=True)
 train_targets = train.label.values
valid_targets = valid.label.values
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index<feature_engineering>
train_dataset=CassavaDataset(
 df=train,
imfolder=TRAIN_IMAGES_DIR,
train=True,
transforms=train_augs 
)
 valid_dataset=CassavaDataset(
 df=valid,
imfolder=TRAIN_IMAGES_DIR,
train=True,
transforms=valid_augs
)
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )<filter>
train_loader = DataLoader(
 train_dataset,
batch_size=16,
num_workers=4,
shuffle=True, 
)
 valid_loader = DataLoader(
 valid_dataset,
batch_size=16,
num_workers=4,
shuffle=False, 
)
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]<sort_values>
def train_model(datasets, dataloaders, model, criterion, optimizer, scheduler, num_epochs, device):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs-1))
print('-' * 10)
 for phase in ['train', 'valid']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0.0
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
 labels=labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
model=model.to(device)
 outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() *inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 if phase == 'train':
scheduler.step()
epoch_loss = running_loss/len(datasets[phase])
 epoch_acc = running_corrects.double() /len(datasets[phase])
 print('{} Loss: {:.4f} Acc: {:.4f}'.format(
 phase, epoch_loss, epoch_acc))
if phase == 'valid' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
 print()
time_elapsed = time.time() -since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
model.load_state_dict(best_model_wts)
 return model
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]<count_values>
datasets={'train':train_dataset,'valid':valid_dataset}
dataloaders={'train':train_loader,'valid':valid_loader}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = VisionTransformer.from_name('ViT-B_16', num_classes=5)
 optimizer = torch.optim.AdamW(model.parameters() , lr=1e-4, weight_decay=0.001)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)
 criterion=nn.CrossEntropyLoss()
num_epochs=6
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )<filter>
model.load_state_dict(torch.load('.. /input/vit-model-1/ViT-B_16.pt'))
train[train['title'].str.contains('Furious')]['title'] 
<filter>
test_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 image_path = ".. /input/cassava-leaf-disease-classification/test_images/"
test_targets = test_df.label.values
test_aug = albu.Compose([
albu.CenterCrop(512, 512, p=1.) ,
albu.Resize(384, 384),
albu.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0),
ToTensorV2() ], p=1.)
test_dataset=CassavaDataset(
 df=test_df,
imfolder=image_path,
train=False,
transforms=test_aug 
)
 test_loader = DataLoader(
 test_dataset,
batch_size=4,
num_workers=4,
shuffle=False,
)
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']<drop_column>
predictions=[]
for imgs in test_loader:
imgs = imgs.to(device)
 with torch.no_grad() :
model=model.to(device)
 outputs = model(imgs)
 _, predicted = torch.max(outputs, dim=1)
 predicted=predicted.to('cpu')
 predictions.append(predicted)
d.remove(d[7] )<feature_engineering>
test_df['label'] = np.concatenate(predictions)
<feature_engineering><EOS> 
test_df.to_csv('submission.csv', index=False )