Kaggle competition: Histopathologic Cancer Detection
bin_cols = [col for col in train.columns.values if col.startswith('bin')]
nom_cols = [col for col in train.columns.values if col.startswith('nom')]
ord_cols = [col for col in train.columns.values if col.startswith('ord')]<categorify>
%matplotlib inline
def ord_to_num(df, col):
keys=np.sort(df[col].unique())
 values=np.arange(len(keys))
map = dict(zip(keys, values))
df[col] = df[col].replace(map )<concatenate>
class CancerDataset(Dataset):
def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.ToTensor() ]), labels_dict={}):
self.datafolder = datafolder
self.datatype = datatype
self.image_files_list = [s for s in os.listdir(datafolder)]
self.transform = transform
self.labels_dict = labels_dict
if self.datatype == 'train':
self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]
else:
self.labels = [0 for _ in range(len(self.image_files_list)) ]
def __len__(self):
return len(self.image_files_list)
 def __getitem__(self, idx):
img_name = os.path.join(self.datafolder, self.image_files_list[idx])
 image = Image.open(img_name)
 image = self.transform(image)
 img_name_short = self.image_files_list[idx].split('.')[0]
if self.datatype == 'train':
label = self.labels_dict[img_name_short]
else:
label = 0
return image, label
ord_to_num(test, 'ord_3')
 ord_to_num(train, 'ord_3' )<concatenate>
IMAGE_NOT_FOUND_COUNTER = 0
labels = pd.read_csv('.. /input/train_labels.csv')
 data_transforms = transforms.Compose([
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5))
])
 data_transforms_test = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5))
])
 tr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.1)
 print("number of training data: ",len(tr))
print("number of testing data: ",len(val))
img_class_dict = {k:v for k, v in zip(labels.id, labels.label)}
train_sampler = SubsetRandomSampler(list(tr.index))
valid_sampler = SubsetRandomSampler(list(val.index))
batch_size = 256
num_workers = 0
dataset = CancerDataset(datafolder='.. /input/train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)
 test_set = CancerDataset(datafolder='.. /input/test/', datatype='test', transform=data_transforms_test)
 train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)
 valid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers )
ord_to_num(train, 'ord_4')
 ord_to_num(test,'ord_4' )<concatenate>
class Flatten(nn.Module):
def forward(self, x):
N, C, H, W = x.size()
return x.view(N, -1 )
ord_to_num(train, 'ord_5')
 ord_to_num(test,'ord_5' )<define_search_space>
avg_loss_list = []
acc_list = []
def train(model, train_loader ,loss_fn, optimizer, num_epochs = 1):
total_loss =0
for epoch in range(num_epochs):
print('Starting epoch %d / %d' %(epoch + 1, num_epochs))
model.train()
for t,(x, y)in enumerate(train_loader):
x_var = Variable(x.type(gpu_dtype))
y_var = Variable(y.type(gpu_dtype ).long())
 scores = model(x_var)
 loss = loss_fn(scores, y_var)
 total_loss += loss.data
if(t + 1)% print_every == 0:
avg_loss = total_loss/print_every
print('t = %d, avg_loss = %.4f' %(t + 1, avg_loss))
avg_loss_list.append(avg_loss)
 total_loss = 0
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc = check_accuracy(fixed_model_gpu, valid_loader)
 print('acc = %f' %(acc))
def check_accuracy(model, loader):
print('Checking accuracy on test set')
 num_correct = 0
num_samples = 0
model.eval()
for x, y in loader:
x_var = Variable(x.type(gpu_dtype))
scores = model(x_var)
 _, preds = scores.data.cpu().max(1)
 num_correct +=(preds == y ).sum()
num_samples += preds.size(0)
 acc = float(num_correct)/ num_samples
acc_list.append(acc)
 return acc
print('Got %d / %d correct(%.2f)' %(num_correct, num_samples, 100 * acc))
values_ord_1=[3,4,0,1,2]<categorify>
print_every = 100
gpu_dtype = torch.cuda.FloatTensor
out_1 = 32
out_2 = 64
out_3 = 128
out_4 = 256
k_size_1 = 3
padding_1 = 1
num_epochs = 10
fixed_model_base = nn.Sequential(
 nn.Conv2d(3, out_1, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_1),
nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_1),
nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_1),
nn.MaxPool2d(2, stride=2),
nn.Conv2d(out_1 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_2),
nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_2),
nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_2),
nn.MaxPool2d(2, stride=2),
nn.Conv2d(out_2 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_3),
nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_3),
nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_3),
nn.MaxPool2d(2, stride=2),
nn.Conv2d(out_3 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_4),
nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_4),
nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1),
nn.ReLU(inplace=True),
nn.BatchNorm2d(out_4),
nn.MaxPool2d(2, stride=2),
Flatten() ,
nn.Linear(9216,512),
nn.ReLU(inplace=True),
nn.Linear(512,10),
nn.ReLU(inplace=True),
nn.Linear(10,2), 
)
 fixed_model_gpu = fixed_model_base.type(gpu_dtype)
 print(fixed_model_gpu)
 loss_fn = nn.modules.loss.CrossEntropyLoss()
optimizer = optim.Adamax(fixed_model_gpu.parameters() , lr = 1e-3)
 train(fixed_model_gpu, train_loader ,loss_fn, optimizer, num_epochs=num_epochs)
 check_accuracy(fixed_model_gpu, valid_loader )
map_ord_1 = dict(zip(keys_ord_1, values_ord_1))
map_ord_1<feature_engineering>
print(avg_loss_list,acc_list )
train['ord_1'] = train['ord_1'].replace(map_ord_1)
 test['ord_1'] = test['ord_1'].replace(map_ord_1 )<define_search_space>
fixed_model_gpu.eval()
preds = []
for batch_i,(data, target)in enumerate(test_loader):
data, target = data.cuda() , target.cuda()
output = fixed_model_gpu(data)
 pr = output[:,1].detach().cpu().numpy()
for i in pr:
preds.append(i)
 test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})
 test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
 data_to_submit = pd.read_csv('.. /input/sample_submission.csv')
 data_to_submit = pd.merge(data_to_submit, test_preds, left_on='id', right_on='imgs')
 data_to_submit = data_to_submit[['id', 'preds']]
data_to_submit.columns = ['id', 'label']
data_to_submit.head()
<categorify><EOS> 
data_to_submit.to_csv('csv_to_submit.csv', index = False )