Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<categorify>
import numpy as np
import pandas as pd
import os
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3,
"Master": 5, "Dr": 4, "Rev": 4, "Col": 4, "Major": 4, "Mlle": 4,"Countess": 4,
"Ms": 4, "Lady": 4, "Jonkheer": 4, "Don": 4, "Dona" : 4, "Mme": 4,"Capt": 4,"Sir": 4 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )<feature_engineering>
torch.cuda.set_device(0)
 torch.cuda.set_device(0 )
for dataset in train_test_data:
dataset['Last_Name'] = dataset['Name'].apply(lambda x: str.split(x, ",")[0] )<concatenate>
MASKS = 'train.csv'
SUB = 'sample_submission.csv'
TRAIN = Path('train/')
 TEST = Path('test/')
 PATH = Path('/kaggle/input/')
 TMP = Path('/kaggle/working/tmp/')
 MODEL = Path('/kaggle/working/model/')
 seg = pd.read_csv(PATH/MASKS ).set_index('Id')
 sample_sub = pd.read_csv(PATH/SUB ).set_index('Id')
 sample= 31072
seg.head()
total_data = train.append(test, sort=False )<correct_missing_values>
train_names_png = [TRAIN/f for f in os.listdir(PATH/TRAIN)]
train_names = list(seg.index.values)
 train_names_sample = list(seg.index.values)[0:sample]
test_names_png = [TEST/f for f in os.listdir(PATH/TEST)]
test_names = list(sample_sub.index.values)
 test_names_sample = list(sample_sub.index.values)[0:sample]
len(train_names_sample), len(test_names )
def fill_missing_cabin(data, match_data):
cabin_list = []
for i in range(len(data)) :
if data.Cabin[i] is np.nan:
cabin_mode = match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Cabin.mode()
if len(cabin_mode)> 0:
cabin_list.append(cabin_mode[0])
 else:
cabin_list.append(np.nan)
 else:
cabin_list.append(data.Cabin[i])
 return cabin_list<categorify>
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return img
train['Cabin'] = fill_missing_cabin(train, total_data)
 test['Cabin'] = fill_missing_cabin(test, total_data )<feature_engineering>
seg2 = seg.iloc[0:sample]
val_idxs = get_cv_idxs(sample )
for dataset in train_test_data:
dataset['Cabin_letter'] = dataset['Cabin'].str[:1].fillna('Missing_cabin_info' )<count_values>
indexes = seg2.Target.apply(str.split)
 y = np.zeros(( sample, 28))
for i in range(sample):
y[i,np.array(indexes[i], dtype=int)]=1
def get_cabin_count(data, column=''):
cabin_count_list=[]
for i in range(len(data)) :
if data[column][i] is np.nan:
cabin_count_list.append(0)
 else:
cabin_count_list.append(data[column][i].strip().count(' ')+1)
 return cabin_count_list<feature_engineering>
((val_x,trn_x),(val_y,trn_y)) = split_by_idx(val_idxs, np.array(train_names_sample), y )
for dataset in train_test_data:
dataset['Cabin_count'] = get_cabin_count(dataset, column='Cabin' )<categorify>
def get_data(sz=128, bs=32):
datasets = ImageData.get_ds(CustomDataset,(trn_x,trn_y),(val_x,val_y), sz=sz, tfms=(None,None), path=PATH/TRAIN)
 datasets[4] = CustomDataset(test_names, test_names, None, PATH/TEST, sz)
 return ImageData(PATH, datasets, bs=bs, num_workers=4, classes=28 )
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )<define_variables>
md = get_data(128 )
categorical_features = ['Pclass', 'Sex', 'Cabin_letter', 'Embarked', 'Title', 'Family_size', 'Parch', 'SibSp']<create_dataframe>
class ConvBN(nn.Module):
"convolutional layer then batchnorm"
def __init__(self, ch_in, ch_out, kernel_size = 3, stride=1, padding=0):
super().__init__()
self.conv = nn.Conv2d(ch_in, ch_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)
 self.bn = nn.BatchNorm2d(ch_out, momentum=0.01)
 self.relu = nn.LeakyReLU(0.1, inplace=True)
 def forward(self, x): return self.relu(self.bn(self.conv(x)) )
for column in categorical_features:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived',\
negative_state='Dead' )<categorify>
class DarknetBlock(nn.Module):
def __init__(self, ch_in):
super().__init__()
ch_hid = ch_in//2
self.conv1 = ConvBN(ch_in, ch_hid, kernel_size=1, stride=1, padding=0)
 self.conv2 = ConvBN(ch_hid, ch_in, kernel_size=3, stride=1, padding=1)
 def forward(self, x): return self.conv2(self.conv1(x)) + x
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )<groupby>
class Darknet(nn.Module):
"Replicates the darknet classifier from the YOLOv3 paper(table 1)"
def make_group_layer(self, ch_in, num_blocks, stride=1):
layers = [ConvBN(ch_in,ch_in*2,stride=stride)]
for i in range(num_blocks): layers.append(DarknetBlock(ch_in*2))
return layers
def __init__(self, num_blocks, num_classes=1000, start_nf=32):
super().__init__()
nf = start_nf
layers = [ConvBN(4, nf, kernel_size=3, stride=1, padding=1)]
for i,nb in enumerate(num_blocks):
layers += self.make_group_layer(nf, nb, stride=(1 if i==1 else 2))
nf *= 2
layers += [nn.AdaptiveAvgPool2d(1), Flatten() , nn.Linear(nf, num_classes)]
self.layers = nn.Sequential(*layers)
 def forward(self, x): return self.layers(x )
test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True )<define_variables>
m = Darknet([1, 2, 4, 4, 3], 28 ).cuda()
continuous_features = [ 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Family_size']<create_dataframe>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
for column in continuous_features:
get_continuous_feature_info(train, column=column, y_column='Survived' )<data_type_conversions>
def f1_(preds, targs, start=0.17, end=0.24, step=0.01):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 return max([fbeta_score(targs,(preds>th), 1, average='samples')
 for th in np.arange(start,end,step)] )
for dataset in train_test_data:
dataset['Cabin_letter'] = dataset['Cabin_letter'].fillna('Missing_cabin_info' )<feature_engineering>
learn = Learner.from_model_data(m, md, tmp_name=TMP, models_name=MODEL)
 learn.crit = FocalLoss()
learn.opt_fn = optim.Adam
learn.metrics = [f1_]
for dataset in train_test_data:
dataset['Cabin_count'] = dataset['Cabin_count'].fillna(0 )<feature_engineering>
lr = 1E-2
for column in ['Cabin_letter', 'Cabin_count']:
get_categorical_feature_info(train, column=column, y_column='Survived' )<drop_column>
learn.fit(lr,1,cycle_len=30,use_clr_beta=(10,10, 0.85, 0.9))
<create_dataframe><EOS> 
learn.save('128' )