Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<categorify>
torch.cuda.set_device(0)
 torch.cuda.set_device(0 )
test_df['Fare'] = test_df['Fare'].fillna(test_df.groupby('Pclass')['Fare'].transform('median'))
train_df['Fare'], test_df['Fare'] = scale_feature('Fare')
 train_df.head()<categorify>
ls /kaggle/input/
train_df['Embarked'] = train_df['Embarked'].fillna('S')
 train_df, test_df = concat_dummies('Embarked')
 train_df.head()<drop_column>
MASKS = 'train.csv'
SUB = 'sample_submission.csv'
TRAIN = Path('train/')
 TEST = Path('test/')
 PATH = Path('/kaggle/input/human-protein-atlas-image-classification/')
 TMP = Path('/kaggle/working/tmp/')
 MODEL = Path('/kaggle/working/model/')
 PRETRAINED = '/kaggle/input/4-channel-darknet-sz-256/model/256'
seg = pd.read_csv(PATH/MASKS ).set_index('Id')
 sample_sub = pd.read_csv(PATH/SUB ).set_index('Id')
 sample= len(seg)
 seg.head()
train_df = train_df.drop(['PassengerId', 'Name', 'Age', 'Ticket', 'Cabin', 'Embarked',
'Title', 'AgeGroup'], axis=1)
 test_df = test_df.drop(['PassengerId', 'Name', 'Age', 'Ticket', 'Cabin', 'Embarked',
'Title', 'AgeGroup'], axis=1)
 train_df.head()<split>
train_names_png = [TRAIN/f for f in os.listdir(PATH/TRAIN)]
train_names = list(seg.index.values)
 train_names_sample = list(seg.index.values)[0:sample]
test_names_png = [TEST/f for f in os.listdir(PATH/TEST)]
test_names = list(sample_sub.index.values)
 test_names_sample = list(sample_sub.index.values)[0:sample]
len(train_names_sample), len(test_names )
X = train_df.iloc[:, 1:].values
y = train_df['Survived'].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=1)
 X_test = test_df.values<choose_model_class>
seg2 = seg.iloc[0:sample]
val_idxs = get_cv_idxs(sample )
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dropout(0.25, input_shape=[20]))
model.add(tf.keras.layers.Dense(25, activation='relu'))
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
 model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 model.summary()<train_model>
indexes = seg2.Target.apply(str.split)
 y = np.zeros(( sample, 28))
for i in range(sample):
y[i,np.array(indexes[i], dtype=int)]=1
history = model.fit(
 x=X_train,
y=y_train,
epochs=10,
batch_size=1,
validation_data=(X_val, y_val)
 )<prepare_output>
((val_x,trn_x),(val_y,trn_y)) = split_by_idx(val_idxs, np.array(train_names_sample), y )
prediction = model.predict(X_test)
 rounded_prediction = np.where(prediction >= 0.5, 1, 0 ).flatten()
submission_df = pd.DataFrame({
'PassengerId': passenger_ids,
'Survived': rounded_prediction
})
 submission_df.head()<save_to_csv>
def get_data(sz=128, bs=32):
datasets = ImageData.get_ds(CustomDataset,(trn_x,trn_y),(val_x,val_y), sz=sz, tfms=(None,None), path=PATH/TRAIN)
 datasets[4] = CustomDataset(test_names, test_names, None, PATH/TEST, sz)
 return ImageData(PATH, datasets, bs=bs, num_workers=4, classes=28 )
submission_df.to_csv('submission.csv', index=False )<set_options>
class ConvBN(nn.Module):
"convolutional layer then batchnorm"
def __init__(self, ch_in, ch_out, kernel_size = 3, stride=1, padding=0):
super().__init__()
self.conv = nn.Conv2d(ch_in, ch_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)
 self.bn = nn.BatchNorm2d(ch_out, momentum=0.01)
 self.relu = nn.LeakyReLU(0.1, inplace=True)
 def forward(self, x): return self.relu(self.bn(self.conv(x)) )
warnings.filterwarnings('ignore')
 %matplotlib inline
%config InlineBackend.figure_format = 'retina' 
<load_from_csv>
class DarknetBlock(nn.Module):
def __init__(self, ch_in):
super().__init__()
ch_hid = ch_in//2
self.conv1 = ConvBN(ch_in, ch_hid, kernel_size=1, stride=1, padding=0)
 self.conv2 = ConvBN(ch_hid, ch_in, kernel_size=3, stride=1, padding=1)
 def forward(self, x): return self.conv2(self.conv1(x)) + x
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv" )<define_variables>
class Darknet(nn.Module):
"Replicates the darknet classifier from the YOLOv3 paper(table 1)"
def make_group_layer(self, ch_in, num_blocks, stride=1):
layers = [ConvBN(ch_in,ch_in*2,stride=stride)]
for i in range(num_blocks): layers.append(DarknetBlock(ch_in*2))
return layers
def __init__(self, num_blocks, num_classes=1000, start_nf=32):
super().__init__()
nf = start_nf
layers = [ConvBN(4, nf, kernel_size=3, stride=1, padding=1)]
for i,nb in enumerate(num_blocks):
layers += self.make_group_layer(nf, nb, stride=(1 if i==1 else 2))
nf *= 2
layers += [nn.AdaptiveAvgPool2d(1), Flatten() , nn.Linear(nf, num_classes)]
self.layers = nn.Sequential(*layers)
 def forward(self, x): return self.layers(x )
passengerid = test.PassengerId<sort_values>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
def f1_(preds, targs, start=0.17, end=0.24, step=0.01):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 return max([fbeta_score(targs,(preds>th), 1, average='samples')
 for th in np.arange(start,end,step)] )
train[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
m = Darknet([1, 2, 4, 4, 3], 28 ).cuda()
train[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
md = get_data(512,8 )
all_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
learn = Learner.from_model_data(m, md, tmp_name=TMP, models_name=MODEL)
 learn.crit = FocalLoss()
learn.opt_fn = optim.Adam
learn.metrics = [f1_]
train[['Survived','Fare']].groupby(['Survived'], as_index=False ).mean().sort_values(by='Fare', ascending=False )<sort_values>
learn.load(PRETRAINED )
train[['Survived','Age']].groupby(['Survived'], as_index=False ).mean().sort_values(by='Age', ascending=False )<categorify>
lr = 1E-2
plt.imshow(train.corr() , cmap='hot', interpolation='nearest')
 plt.show()<sort_values>
learn.fit(lr/10,1,cycle_len=1,use_clr_beta=(10,10, 0.85, 0.9))
def missing_percentage(df):
total = df.isnull().sum().sort_values(ascending = False)
 percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)
 return pd.concat([total, percent], axis=1, keys=['Total','Percent'] )<count_missing_values>
learn.save('512' )
missing_percentage(train )<count_missing_values>
learn.load('512' )
missing_percentage(test )<create_dataframe>
p_v, t_v = learn.predict_with_targs()
def percent_value_counts(df, feature):
percent = pd.DataFrame(round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2))
total = pd.DataFrame(df.loc[:,feature].value_counts(dropna=False))
total.columns = ['Total']
percent.columns = ['Percent']
return pd.concat([total, percent], axis=1 )<count_values>
def sigmoid(a):
return 1/(1+np.exp(-a))
percent_value_counts(train, 'Embarked' )<filter>
sp_v = sigmoid(p_v )
train.loc[train['Embarked'].isnull() ,:]<data_type_conversions>
def f1_np(y_pred, y_true, threshold=0.5):

 y_pred =(y_pred>threshold ).astype(int)
 TP =(y_pred*y_true ).sum(1)
 prec = TP/(y_pred.sum(1)+1e-7)
 rec = TP/(y_true.sum(1)+1e-7)
 res = 2*prec*rec/(prec+rec+1e-7)
 return res.mean()
def f1_n(y_pred, y_true, thresh, n, default=0.5):

 threshold = default * np.ones(y_pred.shape[1])
 threshold[n]=thresh
return f1_np(y_pred, y_true, threshold)
 def find_thresh(y_pred, y_true):

 ths = []
for i in range(y_pred.shape[1]):
aux = []
for th in np.linspace(0,1,100):
aux += [f1_n(y_pred, y_true, th, i)]
ths += [np.array(aux ).argmax() /100]
return np.array(ths )
train["Embarked"].fillna('C',inplace=True )<filter>
ths = find_thresh(sp_v, t_v); ths
test.loc[test['Fare'].isnull() ,:]<categorify>
f1_np(sp_v, t_v, 0.5), f1_np(sp_v, t_v, ths )
missing_value = test.loc[(test['Pclass']==3)&(test['Embarked']=='S')&(test['Sex']=="male"),'Fare'].mean()
test['Fare'].fillna(missing_value, inplace=True )<drop_column>
preds = learn.predict(is_test=True )
train = train.drop('Cabin',axis=1)
 test = test.drop("Cabin", axis=1 )<count_missing_values>
preds = sigmoid(preds)
 threshold = ths
print(preds.shape)
 classes = np.array([str(n)for n in range(28)])
 res = np.array([" ".join(classes[(np.where(pp>threshold)) ])for pp in preds] )
missing_percentage(train )<count_missing_values>
filenames = np.array([os.path.basename(fn ).split('.')[0] for fn in test_names] )
missing_percentage(test )<feature_engineering>
frame = pd.DataFrame(np.array([filenames, res] ).T, columns = ['Id','Predicted'] )
<categorify><EOS> 
frame.to_csv('submission.csv', index=False )