Kaggle competition: iWildCam 2019 - FGVC6
<SOS> metric: MacroFScore Kaggle data source: iwildcam-2019-fgvc6<prepare_output>
import os
from glob import glob
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cv2
predicted_class_indices=np.argmax(predict,axis=1 )<load_pretrained>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train_dir = '.. /input/train_images'
test_dir = '.. /input/test_images'
with open('.. /input/iwildcam2020-classes-dict/cid_invert_dict.pkl', mode='rb')as fin:
cid_invert_dict=pickle.load(fin )<categorify>
print('Total images for train {0}'.format(len(os.listdir(train_dir))))
print('Total images for test {0}'.format(len(os.listdir(test_dir))))
def transform(x):
return cid_invert_dict[str(x)]<save_to_csv>
classes_wild = {0: 'empty', 1: 'deer', 2: 'moose', 3: 'squirrel', 4: 'rodent', 5: 'small_mammal', \
6: 'elk', 7: 'pronghorn_antelope', 8: 'rabbit', 9: 'bighorn_sheep', 10: 'fox', 11: 'coyote', \
12: 'black_bear', 13: 'raccoon', 14: 'skunk', 15: 'wolf', 16: 'bobcat', 17: 'cat',\
18: 'dog', 19: 'opossum', 20: 'bison', 21: 'mountain_goat', 22: 'mountain_lion'}
train_df['classes_wild'] = train_df['category_id'].apply(lambda cw: classes_wild[cw] )
sam_sub_df["Category"] = predicted_class_indices
sam_sub_df["Category"]=sam_sub_df["Category"].apply(transform)
 sam_sub_df = sam_sub_df.loc[:,["Id", "Category"]]
sam_sub_df.to_csv("submission.csv",index=False)
 sam_sub_df.head()<set_options>
train_df['classes_wild'].value_counts()
ImageFile.LOAD_TRUNCATED_IMAGES = True
%matplotlib inline<define_search_model>
import torch
import torch.nn as nn
import torchvision
from torchvision.transforms import transforms
from torch.utils.data import Dataset, DataLoader
from torchvision import models
from sklearn.model_selection import train_test_split
def kaggle_commit_logger(str_to_log, need_print = True):
if need_print:
print(str_to_log)
 os.system('echo ' + str_to_log )<load_from_disk>
train_df = train_df[['file_name','category_id']]
train_df.head()
with open(r'/kaggle/input/iwildcam-2020-fgvc7/iwildcam2020_train_annotations.json')as json_file:
train_data = json.load(json_file )<prepare_output>
category = train_df['category_id'].unique()
encoder = dict([(v, k)for v, k in zip(category, range(len(category)))])
 decoder = dict([(v, k)for k, v in encoder.items() ])
 print(pd.DataFrame({
'Before encoding': list(encoder.keys()),
'After encoding': list(encoder.values())} ).to_string(index=False))
def encoding(labels):
return encoder[int(labels)]
df_train = pd.DataFrame({'id': [item['id'] for item in train_data['annotations']],
'category_id': [item['category_id'] for item in train_data['annotations']],
'image_id': [item['image_id'] for item in train_data['annotations']],
'file_name': [item['file_name'] for item in train_data['images']]})
 df_train.head()<create_dataframe>
train_df['category_id'] = train_df['category_id'].apply(encoding)
 train_df['category_id'].value_counts()
df_image = pd.DataFrame.from_records(train_data['images'])
 indices = []
for _id in df_image[df_image['location'] == 537]['id'].values:
indices.append(df_train[ df_train['image_id'] == _id ].index)
 for the_index in indices:
df_train = df_train.drop(df_train.index[the_index] )<load_pretrained>
class WildDataset(Dataset):
def __init__(self, df, img_dir, transforms=None):
self.df = df
self.img_dir = img_dir
self.transforms = transforms
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
img_name = os.path.join(self.img_dir,
self.df.iloc[idx, 0])
 image = cv2.imread(img_name)
 label = self.df.iloc[idx, 1]
if self.transforms is not None:
image = self.transforms(image)
 return image, label
%%time
indices = []
for i in df_train['file_name']:
try:
Image.open('/kaggle/input/iwildcam-2020-fgvc7/train/' + i)
 except:
print(i)
 df_train.drop(df_train.loc[df_train['file_name']==i].index, inplace=True )<load_from_disk>
train, val = train_test_split(train_df, stratify=train_df.category_id, test_size=0.1)
 len(train), len(val )
with open(r'/kaggle/input/iwildcam-2020-fgvc7/iwildcam2020_test_information.json')as f:
test_data = json.load(f )<prepare_output>
aug = transforms.Compose([transforms.ToPILImage() ,
transforms.Resize(( 32, 32)) ,
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
 ])
df_test = pd.DataFrame.from_records(test_data['images'])
 df_test.head()<define_variables>
dataset_train = WildDataset(df=train,
img_dir=train_dir,
transforms=aug)
 dataset_valid = WildDataset(df=val,
img_dir=train_dir,
transforms=aug)
 train_loader = DataLoader(dataset=dataset_train, batch_size=24, shuffle=True)
 val_loader = DataLoader(dataset_valid, batch_size=24, shuffle=False, num_workers=0 )
batch_size = 64
IMG_SIZE = 64
N_EPOCHS = 1
ID_COLNAME = 'file_name'
ANSWER_COLNAME = 'category_id'
TRAIN_IMGS_DIR = r'.. /input/iwildcam-2020-fgvc7/train/'
TEST_IMGS_DIR = r'.. /input/iwildcam-2020-fgvc7/test/'<split>
num_epochs = 2
num_classes = 14
learning_rate = 0.02
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )
train_df, test_df = train_test_split(df_train[[ID_COLNAME, ANSWER_COLNAME]],
test_size = 0.15,
shuffle = True
)<define_variables>
def conv3x3(in_channels, out_channels, stride=1):
return nn.Conv2d(in_channels, out_channels, kernel_size=3,
stride=stride, padding=1, bias=False)
 class ResidualBlock(nn.Module):
def __init__(self, in_channels, out_channels, stride=1, downsample=None):
super(ResidualBlock, self ).__init__()
self.conv1 = conv3x3(in_channels, out_channels, stride)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(out_channels, out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.downsample = downsample
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=14):
super(ResNet, self ).__init__()
self.in_channels = 16
self.conv = conv3x3(3, 16)
 self.bn = nn.BatchNorm2d(16)
 self.relu = nn.LeakyReLU(inplace=True)
 self.layer1 = self.make_layer(block, 16, layers[0])
 self.layer2 = self.make_layer(block, 32, layers[1], 2)
 self.layer3 = self.make_layer(block, 64, layers[2], 2)
 self.layer4 = self.make_layer(block, 128, layers[3], 2)
 self.avg_pool = nn.AdaptiveAvgPool2d(4)
 self.fc = nn.Linear(128, num_classes)
 def make_layer(self, block, out_channels, blocks, stride=1):
downsample = None
if(stride != 1)or(self.in_channels != out_channels):
downsample = nn.Sequential(
 conv3x3(self.in_channels, out_channels, stride=stride),
nn.BatchNorm2d(out_channels))
layers = []
layers.append(block(self.in_channels, out_channels, stride, downsample))
self.in_channels = out_channels
for i in range(1, blocks):
layers.append(block(out_channels, out_channels))
return nn.Sequential(*layers)
 def forward(self, x):
out = self.conv(x)
 out = self.bn(out)
 out = self.relu(out)
 out = self.layer1(out)
 out = self.layer2(out)
 out = self.layer3(out)
 out = self.layer4(out)
 out = self.avg_pool(out)
 out = out.view(out.size(0), -1)
 out = self.fc(out)
 return out
def create_resnet_model(output_dim: int = 1)-> nn.Module:
model = ResNet(ResidualBlock, [2, 2, 2, 2])
 in_features = model.fc.in_features
model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.fc = nn.Linear(in_features, output_dim)
 model = model.to(device)
 return model
model = create_resnet_model(output_dim=num_classes)
 criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adamax(model.parameters() , lr=learning_rate )
NUM_CLASSES = len(CLASSES_TO_USE)
 NUM_CLASSES<define_variables>
total_step = len(train_loader)
 for epoch in range(num_epochs):
for i,(images, labels)in enumerate(train_loader):
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 loss = criterion(outputs, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(i+1)% 100 == 0:
print("Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}" 
.format(epoch+1, num_epochs, i+1, total_step, loss.item()))
CLASSMAP = dict(
 [(i, j)for i, j
in zip(CLASSES_TO_USE, range(NUM_CLASSES))
] 
)
 CLASSMAP<define_variables>
model.eval()
with torch.no_grad() :
correct = 0
total = 0
for images, labels in val_loader:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
print('Accuracy of the model on the 19630 test images: {} %'.format(100 * correct / total))
REVERSE_CLASSMAP = dict([(v, k)for k, v in CLASSMAP.items() ])
 REVERSE_CLASSMAP<choose_model_class>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['Id'] = sub['Id'] + '.jpg'
sub.head()
model = models.densenet121(pretrained='imagenet' )<choose_model_class>
dataset_valid = WildDataset(df=sub,
img_dir=test_dir,
transforms=aug)
 test_loader = DataLoader(dataset_valid, batch_size=24, shuffle=False )
new_head = torch.nn.Linear(model.classifier.in_features, NUM_CLASSES)
 model.classifier = new_head<load_from_disk>
model.eval()
preds = []
for images, labels in test_loader:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 predicted
for i in predicted.detach().cpu().numpy() :
preds.append(i)
model.load_state_dict(torch.load('.. /input/iwild2020-torch/model'))<categorify>
sub['Predicted'] = preds
sub['Id'] = sub['Id'].str[:-4]
sub.head()
normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 train_augmentation = transforms.Compose([
transforms.Resize(( IMG_SIZE,IMG_SIZE)) ,
transforms.ToTensor() ,
normalizer,
])
 val_augmentation = transforms.Compose([
transforms.Resize(( IMG_SIZE,IMG_SIZE)) ,
transforms.ToTensor() ,
normalizer,
] )<create_dataframe>
def decoding(labels):
return decoder[int(labels)]
class IMetDataset(Dataset):
def __init__(self,
df,
images_dir,
n_classes = NUM_CLASSES,
id_colname = ID_COLNAME,
answer_colname = ANSWER_COLNAME,
label_dict = CLASSMAP,
transforms = None 
):
self.df = df
self.images_dir = images_dir
self.n_classes = n_classes
self.id_colname = id_colname
self.answer_colname = answer_colname
self.label_dict = label_dict
self.transforms = transforms
def __len__(self):
return self.df.shape[0]
def __getitem__(self, idx):
cur_idx_row = self.df.iloc[idx]
img_id = cur_idx_row[self.id_colname]
img_name = img_id
img_path = os.path.join(self.images_dir, img_name)
 img = Image.open(img_path)
 if self.transforms is not None:
img = self.transforms(img)
 if self.answer_colname is not None:
label = torch.zeros(( self.n_classes,), dtype=torch.float32)
 label[self.label_dict[cur_idx_row[self.answer_colname]]] = 1.0
return img, label
else:
return img, img_id<create_dataframe>
sub['Predicted'] = sub['Predicted'].apply(decoding)
 sub.head()
sub.to_csv('submission.csv', index=False )
train_dataset = IMetDataset(train_df, TRAIN_IMGS_DIR, transforms = train_augmentation)
 test_dataset = IMetDataset(test_df, TRAIN_IMGS_DIR, transforms = val_augmentation )<load_pretrained>
sub['Predicted'].value_counts()