Kaggle competition: RSNA Intracranial Hemorrhage Detection
train = pd.read_csv(f'{PATH}train.csv')
 test = pd.read_csv(f'{PATH}test.csv' )<prepare_x_and_y>
import os
import time
import copy
import glob
import numpy as np
import pandas as pd
from tqdm import tqdm
from ipywidgets import IntProgress
import cv2
from albumentations import Compose, ShiftScaleRotate, Resize
from albumentations.pytorch import ToTensor
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import torchvision
from torchvision import datasets, models, transforms
from torch.utils.data import Dataset, random_split
from matplotlib import pyplot as plt
idx = np.random.permutation(train.shape[0])
 valid_idx, train_idx = idx[:10000], idx[10000:]
xtrain, xvalid = train.iloc[train_idx,:].values, train.iloc[valid_idx,:].values
ytrain, yvalid = xtrain[:, 0], xvalid[:,0]
xtrain, xvalid = xtrain[:, 1:], xvalid[:, 1:]<feature_engineering>
data_path = '.. /input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/'
train_data_images = '.. /input/rsna-train-stage-1-images-png-224x/stage_1_train_png_224x/'
train = pd.read_csv(os.path.join(data_path,'stage_2_train.csv'))
train[['ID','Image','Diagnosis']]=train['ID'].str.split('_',expand= True)
 train = train[['Image','Diagnosis','Label']]
train.drop_duplicates(inplace= True)
 train = train.pivot(index = 'Image' , columns = 'Diagnosis', values = 'Label' ).reset_index()
train['Image'] = 'ID_' + train['Image']
png = glob.glob(os.path.join(train_data_images, '*.png'))
png = [os.path.basename(png)[:-4] for png in png]
png = np.array(png)
 train = train[train['Image'].isin(png)]
train = train[:300000]
train.to_csv('train.csv', index = False)
 print(train.shape )
xtrain =(xtrain-mean)/std
xvalid =(xvalid-mean)/std<normalization>
data_path = '.. /input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/'
test_data_images = '.. /input/stage-2-png/'
test = pd.read_csv(os.path.join(data_path,'stage_2_sample_submission.csv'))
test[['ID','Image','Diagnosis']]=test['ID'].str.split('_',expand= True)
 test['Image'] = 'ID_' + test['Image']
test = test [['Image','Label']]
test.drop_duplicates(inplace= True)
 test.to_csv('test.csv', index = False)
 print(test.shape )
def rotate_cv(im_arr):

 output = []
degs = np.random.randint(-20, 20, len(im_arr))
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
r,c,*_ = img.shape
M = cv2.getRotationMatrix2D(( c//2,r//2),degs[i],1)
 out = cv2.warpAffine(img,M,(c,r))
output.append(out)
 return np.array(output)
 def translate_cv(im_arr):

 output = []
pos = np.random.randint(-7, 7, len(im_arr))
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
r, c, *_ = img.shape
M = np.float32([[1, 0, pos[i]],[0, 1, 0]])
 dst = cv2.warpAffine(img, M,(c,r))
output.append(dst)
 return np.array(output)
 def gaussBlur_cv(im_arr):

 output = []
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
y = int(np.random.randint(3)*0.5)
 if y: dst = cv2.GaussianBlur(img,(0, 0), 2)
 else: dst = img
output.append(dst)
 return np.array(output)
 def sobel_cv(im_arr):

 output = []
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
y = int(np.random.randint(3)*0.5)
 if y: dst = cv2.Sobel(img, cv2.CV_8U, 1, 0)
 else: dst = img
output.append(dst)
 return np.array(output)
 def dilate_cv(im_arr):

 output = []
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
y = int(np.random.randint(3)*0.5)
 kernel = np.ones(( 3,3), np.uint8)
 if y: dst = cv2.dilate(img, kernel, iterations=1)
 else: dst = img
output.append(dst)
 return np.array(output )<load_pretrained>
class RSNA(Dataset):
def __init__(self, csv_file, path, labels, transform=None):
self.path = path
self.data = pd.read_csv(csv_file)
 self.transform = transform
self.labels = labels
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join(self.path, self.data.loc[idx, 'Image'] + '.png')
 img = cv2.imread(img_name)
 if self.transform:
augmented = self.transform(image=img)
 img = augmented['image']
if self.labels:
labels = torch.tensor(
 self.data.loc[idx, ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']])
 return {'image': img, 'labels': labels}
else:
return {'image': img}
class DataLoader() :
def __init__(self, xt, yt, batch_size=32):

 self.xt, self.yt = xt, yt
self.bs = batch_size
def __iter__(self):
xt_trans = translate_cv(dilate_cv(self.xt))
xt_rot = rotate_cv(xt_trans)
 xout, yout = [], []
for i in range(len(self.xt)) :
xout.append(xt_rot[i])
 yout.append(self.yt[i])
 if len(xout)== self.bs:
yield(Variable(torch.cuda.DoubleTensor(np.array(xout, dtype="float32"))),
Variable(torch.cuda.LongTensor(np.array(yout, dtype="float32"))))
xout, yout = [], []
if len(xout)> 0: yield(Variable(torch.cuda.DoubleTensor(np.array(xout, dtype="float32"))),
Variable(torch.cuda.LongTensor(np.array(yout, dtype="float32"))))
def __len__(self):
if len(self.xt)%self.bs==0 : return len(self.xt)//self.bs
else : return len(self.xt)//self.bs+1<load_pretrained>
transform_train = Compose([ShiftScaleRotate() ,ToTensor() ])
 transform_test = Compose([ToTensor() ])
 train_dataset = RSNA(csv_file='train.csv', path=train_data_images, transform=transform_train, labels=True)
 test_dataset = RSNA(csv_file='test.csv', path=test_data_images, transform=transform_test, labels=False)
 batch_size = 64
data_train_generator = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
 data_test_generator = torch.utils.data.DataLoader(dataset=test_dataset,batch_size=batch_size,shuffle=False )
%time dl = DataLoader(xtrain_imgs, ytrain )<categorify>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model0 = models.googlenet(pretrained = True)
 model = torch.nn.Sequential(model0, torch.nn.Linear(1000, 6)).to(device)
class valGenerator() :
def __init__(self, xv, yv, batch_size=500):

 self.data = np.zeros(( len(xv), 28*28+1))
self.bs = batch_size
self.data[:, 0] = yv
self.data[:, 1:785] = xv
def __iter__(self):
while True:
idxs = np.random.permutation(len(self.data)) [0:self.bs]
yield(Variable(torch.cuda.DoubleTensor(self.data[idxs, 1:785].reshape(( -1, 1, 28, 28)))) ,
Variable(torch.cuda.LongTensor(self.data[idxs, 0])) )<compute_test_metric>
num_epochs = 3
optimizer = optim.Adam(model.parameters() , lr=4e-5)
 criterion = torch.nn.BCEWithLogitsLoss()
vgen = valGenerator(xvalid, yvalid )<choose_model_class>
def train_model(model,criterion,optimizer,num_epochs=1):
for epoch in range(1, num_epochs+1):
print('Epoch {}/{}'.format(epoch, num_epochs))
print('-' * 10)
 model.train()
tr_loss = 0
tk0 = tqdm(data_train_generator, desc="Iteration")
 for step, batch in enumerate(tk0):
inputs = batch["image"]
labels = batch["labels"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.float)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
tr_loss += loss.item()
optimizer.step()
optimizer.zero_grad()
torch.save(model.state_dict() , f'resnext50_{epoch}.pth')
 epoch_loss = tr_loss / len(data_train_generator)
 print('Training Loss: {:.4f}'.format(epoch_loss))
return model
class ConvNet(nn.Module):
def __init__(self):
super(ConvNet, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(1, 64, 3, padding=1),
nn.ReLU() ,
nn.Conv2d(64, 64, 3, padding=1),
nn.ReLU() ,
nn.MaxPool2d(2, padding=1),
nn.BatchNorm2d(64),
nn.Dropout(0.25),
nn.Conv2d(64, 32, 3, padding=1),
nn.ReLU() ,
nn.Conv2d(32, 32, 3, padding=1),
nn.ReLU() ,
nn.MaxPool2d(2, padding=1),
nn.Dropout(0.20), 
)
 self.fc = nn.Sequential(
 nn.Linear(32*8*8, 500),
nn.ReLU() ,
nn.BatchNorm1d(500),
nn.Dropout(p=0.5),
nn.Linear(500, 100),
nn.ReLU() ,
nn.Linear(100, 10),
nn.ReLU() ,
nn.Softmax(dim=1)
)
 def forward(self, x):
x = self.conv(x)
 x = x.view(-1, x.size(1)*x.size(2)*x.size(3))
return self.fc(x )<set_options>
model_trained = train_model(model,criterion,optimizer,num_epochs )
def set_lrs(opt, lr):

 for pg in opt.param_groups: pg['lr'] = lr<choose_model_class>
for param in model.parameters() :
param.requires_grad = False
model.eval()
test_pred = np.zeros(( len(test_dataset)* 6, 1))
for i, batch_ in enumerate(tqdm(data_test_generator)) :
batch_ = batch_["image"]
batch_ = batch_.to(device, dtype=torch.float)
 with torch.no_grad() :
pred = model(batch_)
 test_pred[(i * batch_size * 6):(( i + 1)* batch_size * 6)] = torch.sigmoid(
 pred ).detach().cpu().reshape(( len(batch_)* 6, 1))
<prepare_x_and_y><EOS> 
submission = pd.read_csv(os.path.join(data_path, 'stage_2_sample_submission.csv'))
submission = pd.concat([submission.drop(columns=['Label']), pd.DataFrame(test_pred)], axis=1)
 submission.columns = ['ID', 'Label']
submission.to_csv('submission-1.csv', index=False )