Kaggle competition: Hackathon Auto_matic
package_dir = ".. /input/pretrained-models/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, package_dir)
 device = torch.device("cuda:0")
 ImageFile.LOAD_TRUNCATED_IMAGES = True<load_from_csv>
class CarDataset(datasets.ImageFolder):

 def __init__(self,root,csv_file,transform=None):
super(CarDataset,self ).__init__(root,transform)
 self.df = pd.read_csv(csv_file,header=None)
 def __getitem__(self, index):
original_tuple = super(CarDataset, self ).__getitem__(index)
 path = self.imgs[index][0]
label = path.split('/')[5]
if label=='Ram C-V Cargo Van Minivan 2012':
label='Ram C/V Cargo Van Minivan 2012'
name_index = self.df.index[self.df[0]==label][0]
target = torch.zeros([1,196])
 target[0][name_index] = 1
path = path.split('/')[6][0:-4]
tuple_with_path =(original_tuple +(path,)+(name_index,))
return tuple_with_path
class RetinopathyDatasetTest(Dataset):
def __init__(self, csv_file, transform):
self.data = pd.read_csv(csv_file)
 self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join('.. /input/aptos2019-blindness-detection/test_images', self.data.loc[idx, 'id_code'] + '.png')
 image = Image.open(img_name)
 image = self.transform(image)
 return {'image': image}<choose_model_class>
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('CUDA is not available.Training on CPU...')
 else:
print('CUDA is available! Training on GPU...' )
model = pretrainedmodels.__dict__['resnet101'](pretrained=None)
 model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.last_linear = nn.Sequential(
 nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
nn.Dropout(p=0.25),
nn.Linear(in_features=2048, out_features=2048, bias=True),
nn.ReLU() ,
nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
nn.Dropout(p=0.5),
nn.Linear(in_features=2048, out_features=1, bias=True), 
)
 model.load_state_dict(torch.load(".. /input/mmmodel/model.bin"))
model = model.to(device )<find_best_params>
names_df = pd.read_csv('.. /input/names.csv',header=None)
 classes = names_df[0]
for param in model.parameters() :
param.requires_grad = False
model.eval()<load_from_csv>
model = models.vgg16(pretrained=True)
 print(model )
test_transform = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.RandomHorizontalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 test_dataset = RetinopathyDatasetTest(csv_file='.. /input/aptos2019-blindness-detection/sample_submission.csv',
transform=test_transform )<predict_on_test>
valid_size = 0.2
train_transform = transforms.Compose([transforms.Resize(255),
transforms.CenterCrop(224),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5))
])
 test_transform = transforms.Compose([transforms.Resize(255),
transforms.CenterCrop(224),
transforms.ToTensor()
])
 train_data = CarDataset(root=".. /input/car_data/car_data/train",transform=train_transform,csv_file=".. /input/names.csv")
 test_data = CarDataset(root=".. /input/car_data/car_data/test",transform=test_transform,csv_file=".. /input/names.csv")
 num_train = len(train_data)
 indices = list(range(num_train))
np.random.shuffle(indices)
 split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
 valid_sampler = SubsetRandomSampler(valid_idx)
 train_loader = torch.utils.data.DataLoader(train_data, batch_size=20,
sampler=train_sampler)
 valid_loader = torch.utils.data.DataLoader(train_data, batch_size=20,
sampler=valid_sampler)
 test_loader = torch.utils.data.DataLoader(test_data, batch_size=20 )
def tta(test_dataset, batch_size=32, shuffle=False, num_workers=4):
test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
 test_preds = np.zeros(( len(test_dataset), 1))
tk = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk):
x_batch = x_batch['image']
pred = model(x_batch.to(device))
test_preds[i*32:(i+1)*32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 return test_preds<feature_engineering>
n_inputs = model.classifier[6].in_features
last_layer = nn.Linear(n_inputs, len(classes))
model.classifier[6] = last_layer
if train_on_gpu:
model.cuda()
for i in range(1, 6):
locals() ['test_preds{}'.format(i)] = tta(test_dataset )<compute_test_metric>
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.classifier.parameters() , lr=0.01 )
preds2 =(test_preds1 + test_preds2 + test_preds3 + test_preds4 + test_preds5)/ 5.0<prepare_output>
n_epochs = 10
valid_loss_min = np.Inf
for epoch in range(1, n_epochs+1):
train_loss = 0.0
valid_loss = 0.0
model.train()
for batch_i,(data, index, path,target)in enumerate(train_loader):
if train_on_gpu:
data, target = data.cuda() , target.cuda()
optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 loss.backward()
optimizer.step()
train_loss += loss.item()
if batch_i % 20 == 19:
print('Epoch %d, Batch %d Train Loss: %.16f' % 
(epoch, batch_i + 1, train_loss / 20))
train_loss = 0.0
model.eval()
for batch_i,(data, index, path,target)in enumerate(valid_loader):
if train_on_gpu:
data, target = data.cuda() , target.cuda()
output = model(data)
 loss = criterion(output, target)
 valid_loss += loss.item()
if batch_i % 20 == 19:
print('Epoch %d, Batch %d Validation Loss: %.16f' % 
(epoch, batch_i + 1, valid_loss / 20))
valid_loss = 0.0
if valid_loss <= valid_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...'.format(
 valid_loss_min,
valid_loss))
torch.save(model.state_dict() , 'model.pt')
 valid_loss_min = valid_loss
coef = [0.5, 1.5, 2.5, 3.5]
for i, pred in enumerate(preds2):
if pred < coef[0]:
preds2[i] = 0
elif pred >= coef[0] and pred < coef[1]:
preds2[i] = 1
elif pred >= coef[1] and pred < coef[2]:
preds2[i] = 2
elif pred >= coef[2] and pred < coef[3]:
preds2[i] = 3
else:
preds2[i] = 4<import_modules>
model.load_state_dict(torch.load('model.pt'))
import cv2
import matplotlib.pyplot as plt
from os.path import isfile
import torch
import torch.nn as nn
import numpy as np
import pandas as pd
import os
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
import time
from tqdm import tqdm
import random
import sys<define_variables>
results = {}
for data,index,path,target in test_loader:
if train_on_gpu:
data, target = data.cuda() , target.cuda()
output = model(data)
 prob = torch.nn.functional.softmax(output, dim=1)
 _, pred = torch.max(prob, 1)
 for k in range(len(pred)) :
name = path[k]
results[name] = pred[k].cpu().tolist()
package_path = '.. /input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'
sys.path.append(package_path )<import_modules>
output = pd.DataFrame(results, index=[0] ).transpose()
output = output.reset_index()
output.rename(columns={'index':'Id',0:'Predicted'}, inplace=True)
 output['Predicted'] = output['Predicted']+1
output.head()
from efficientnet_pytorch import EfficientNet<set_options>
output['Predicted'].value_counts()
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index=False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html )
seed_everything(1234)
 TTA = 5
num_classes = 1
IMG_SIZE = 384<define_variables>
create_download_link(output )
<train_model><EOS> 
output.to_csv('output.csv',index=False )