Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<filter>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
train = train[train['GarageArea'] < 1200] 
<sort_values>
def open_4_channel(self,fname):
fname = str(fname)
 if fname.endswith('.png'):
fname = fname[:-4]
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(fname+'_'+color+'.png', flags ).astype(np.float32)/255
for color in colors]
x = np.stack(img, axis=-1)
 return Image(pil2tensor(x, np.float32 ).float() )
total = train.isnull().sum().sort_values(ascending = False)
 percent =(train.isnull().sum() /train.isnull().count() ).sort_values(ascending = False)
 missing_data = pd.concat([total,percent], axis = 1, keys = ['Total', 'Percent'])
 missing_data.head(20 )<count_values>
def show_batch_cstm(self, rows:int=5, ds_type:DatasetType=DatasetType.Train, **kwargs)->None:
"Show a batch of data in `ds_type` on a few `rows`."
x,y = self.one_batch(ds_type, True, True)
 x=x[:,:3,:,:]
if self.train_ds.x._square_show: rows = rows ** 2
xs = [self.train_ds.x.reconstruct(grab_idx(x, i, self._batch_first)) for i in range(rows)]
if has_arg(self.train_ds.y.reconstruct, 'x'):
ys = [self.train_ds.y.reconstruct(grab_idx(y, i), x=x)for i,x in enumerate(xs)]
else : ys = [self.train_ds.y.reconstruct(grab_idx(y, i)) for i in range(rows)]
print(ys)
 self.train_ds.x.show_xys(xs, ys, **kwargs)
print("Original:
")
 print(train.Street.value_counts() , "
" )<categorify>
path='.'
input_fldr=Path('.. /input/')
 trn_fldr=f'{input_fldr}/train/'
test_fldr=f'{input_fldr}/test/'
trn_lbl=f'{input_fldr}/train.csv'
sample_csv = f'{input_fldr}/sample_submission.csv'
df_trn=pd.read_csv(f'{input_fldr}/train.csv')
 df_trn.head()
train['enc_street'] = pd.get_dummies(train.Street, drop_first=True)
 test['enc_street'] = pd.get_dummies(train.Street, drop_first=True )<categorify>
sz=512
bs=16
def encode(x): return 1 if x == 'Partial' else 0
train['enc_condition'] = train.SaleCondition.apply(encode)
 test['enc_condition'] = test.SaleCondition.apply(encode )<categorify>
targets=[np.array(target.split(' ')).astype(float)for target in df_trn.Target.values]
mapping=MultiLabelBinarizer()
mapping.fit(targets)
 targets_mapped=mapping.transform(targets )
def encode(x): return 1 if x == 'Ex' else 0
train['enc_kitchen'] = train.SaleCondition.apply(encode)
 test['enc_kitchen'] = test.SaleCondition.apply(encode )<categorify>
X_train, y_train, X_test, y_test = iterative_train_test_split(df_trn.Id[:,None], targets_mapped, test_size = 0.5)
 plt.plot(np.sum(y_train,axis=0)-np.sum(y_test,axis=0),label='iterative')
 X_train, X_test,y_train , y_test = train_test_split(df_trn.Id[:,None], targets_mapped, test_size = 0.5)
 plt.plot(np.sum(y_train,axis=0)-np.sum(y_test,axis=0),label='split')
 plt.legend()
train['enc_utilities'] = pd.get_dummies(train.Utilities, drop_first=True)
 test['enc_utilities'] = pd.get_dummies(train.Utilities, drop_first=True )<categorify>
p=1.
 n=int(p*df_trn.shape[0])
 while True:
if p<1.0:
idx=np.random.choice(len(df_trn.index),n)
 else:
idx=range(len(df_trn.index))
X_train, y_train, X_val, y_val = iterative_train_test_split(df_trn.index[idx,None], targets_mapped[idx,:], test_size = 0.2)
 if(np.sum(targets_mapped[idx,:],axis=0)>0 ).all() and(( X_train.shape[0]%bs!=1)and(X_val.shape[0]%bs!=1)) :
break
X_train.shape,(X_train.shape[0]%bs),X_val.shape,(X_val.shape[0]%bs )
def encode(x): return 1 if x == 'Con' else 0
train['enc_type'] = train.SaleType.apply(encode)
 test['enc_type'] = test.SaleType.apply(encode )<prepare_x_and_y>
name_label_dict = {
0:('Nucleoplasm', 12885),
1:('Nuclear membrane', 1254),
2:('Nucleoli', 3621),
3:('Nucleoli fibrillar center', 1561),
4:('Nuclear speckles', 1858),
5:('Nuclear bodies', 2513),
6:('Endoplasmic reticulum', 1008),
7:('Golgi apparatus', 2822),
8:('Peroxisomes', 53),
9:('Endosomes', 45),
10:('Lysosomes', 28),
11:('Intermediate filaments', 1093),
12:('Actin filaments', 688),
13:('Focal adhesion sites', 537),
14:('Microtubules', 1066),
15:('Microtubule ends', 21),
16:('Cytokinetic bridge', 530),
17:('Mitotic spindle', 210),
18:('Microtubule organizing center', 902),
19:('Centrosome', 1482),
20:('Lipid droplets', 172),
21:('Plasma membrane', 3777),
22:('Cell junctions', 802),
23:('Mitochondria', 2965),
24:('Aggresome', 322),
25:('Cytosol', 8228),
26:('Cytoplasmic bodies', 328),
27:('Rods &amp; rings', 11)
 }
n_labels = 50782
def cls_wts(label_dict, mu=0.5):
prob_dict, prob_dict_bal = {}, {}
max_ent_wt = 1/28
for i in range(28):
prob_dict[i] = label_dict[i][1]/n_labels
if prob_dict[i] > max_ent_wt:
prob_dict_bal[i] = prob_dict[i]-mu*(prob_dict[i] - max_ent_wt)
 else:
prob_dict_bal[i] = prob_dict[i]+mu*(max_ent_wt - prob_dict[i])
 return prob_dict, prob_dict_bal
y = np.log(train.SalePrice)
 X = data.drop(['SalePrice', 'Id'], axis=1 )<split>
ros = RandomOverSampler(random_state=42,sampling_strategy=over_sample)
 X_res, y_res = ros.fit_resample(X_train, np.argmin(-y_train*(to_np(alpha_log)/np.min(to_np(alpha_log))),axis=1))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
<choose_model_class>
ImageItemList.open=open_4_channel
ImageDataBunch.show_batch=show_batch_cstm
regr = linear_model.LinearRegression()
regr.fit(X_train, y_train)
 y_pred = regr.predict(X_test)
 print('The accuracy of the Linear Regression is',r2_score(y_test,y_pred))
print('RMSE is:
', mean_squared_error(y_test, y_pred))<choose_model_class>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=30., max_zoom=1,
max_lighting=0.05, max_warp=0.)
xg_reg = xgb.XGBRegressor(
 colsample_bytree=0.2,
gamma=0.0,
learning_rate=0.05,
max_depth=6,
min_child_weight=1.5,
n_estimators=7200,
reg_alpha=0.9,
reg_lambda=0.6,
subsample=0.2,
seed=42,
silent=1)
 xg_reg.fit(X_train,y_train)
 preds = xg_reg.predict(X_test)
 print('The accuracy of the xgboost is',r2_score(y_test,preds))
print('RMSE is:
', mean_squared_error(y_test,preds))<compute_test_metric>
test_ids = list(sorted({fname.split('_')[0] for fname in os.listdir(input_fldr/'test')}))
test_fnames = [input_fldr/'test'/test_id for test_id in test_ids]
gbr_model = GradientBoostingRegressor().fit(X_train,y_train)
 gbr_preds = gbr_model.predict(X_test)
 print('The accuracy of the Gradient boost is',r2_score(y_test,gbr_preds))
print('RMSE is:
', mean_squared_error(y_test,gbr_preds))<train_model>
src =(ImageItemList.from_csv(input_fldr, 'train.csv', folder='train', suffix='.png',num_workers=0)
.split_by_idxs(X_res[:,0],X_val[:,0])
.label_from_df(sep=' ', classes=[str(i)for i in range(28)]))
src.add_test(test_fnames, label='0');
X_train=pd.get_dummies(X_train)
 X_test=pd.get_dummies(X_test)
 model = BaggingRegressor(tree.DecisionTreeRegressor(random_state=1))
model.fit(X_train, y_train)
 bag_preds = model.predict(X_test)
 print('The accuracy of the Bagging Regressor is',r2_score(y_test,bag_preds))
print('RMSE is:
', mean_squared_error(y_test,bag_preds))<choose_model_class>
data =(src.transform(tfms, size=sz)
.databunch(num_workers=0))
stats=data.batch_stats()
data.normalize(stats )
lgbm_model = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11 ).fit(X_train, y_train)
 lgbm_preds = lgbm_model.predict(X_test)
 print('The accuracy of the Bagging Regressor is',r2_score(y_test,lgbm_preds))
print('RMSE is:
', mean_squared_error(y_test,lgbm_preds))<choose_model_class>
data.show_batch(rows=3, figsize=(12,9))
r_alphas = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
ridge_model = make_pipeline(RobustScaler() , RidgeCV(alphas = r_alphas)).fit(X_train, y_train)
 ridge_preds = ridge_model.predict(X_test)
 print('The accuracy of the ridge Regressor is',r2_score(y_test,ridge_preds))
print('RMSE is:
', mean_squared_error(y_test,ridge_preds)) 
<train_model>
RESNET_ENCODERS = {
34: torchvision.models.resnet34,
50: torchvision.models.resnet50,
101: torchvision.models.resnet101,
152: torchvision.models.resnet152,
}
class Resnet4Channel(nn.Module):
def __init__(self, encoder_depth=34, pretrained=True, num_classes=28):
super().__init__()
encoder = RESNET_ENCODERS[encoder_depth](pretrained=pretrained)
 w = encoder.conv1.weight
self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 self.conv1.weight = nn.Parameter(torch.cat(( w,w[:,:1,:,:]),dim=1))
self.bn1 = encoder.bn1
self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = encoder.layer1
self.layer2 = encoder.layer2
self.layer3 = encoder.layer3
self.layer4 = encoder.layer4
self.avgpool = encoder.avgpool
self.fc = nn.Linear(512 *(1 if encoder_depth==34 else 4), num_classes)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
def resnet34(pretrained):
return Resnet4Channel(encoder_depth=34)
 def resnet50(pretrained):
return Resnet4Channel(encoder_depth=50)
 def _resnet_split(m): return(m[0][6],m[1])
alphas= [0.00005, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005,
0.0006, 0.0007, 0.0008]
lasso_model = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7,
alphas = alphas,
random_state = 42)).fit(X_train, y_train)
 lasso_preds = lasso_model.predict(X_test)
 print('The accuracy of the lasso Regressor is',r2_score(y_test,lasso_preds))
print('RMSE is:
', mean_squared_error(y_test,lasso_preds)) 
<train_on_grid>
f1_score = partial(fbeta, thresh=0.35, beta=1 )
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elastic_cv = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
l1_ratio=e_l1ratio))
elastic_model = elastic_cv.fit(X_train, y_train)
 elastic_preds = elastic_model.predict(X_test)
 print('The accuracy of the lasso Regressor is',r2_score(y_test,elastic_preds))
print('RMSE is:
', mean_squared_error(y_test,elastic_preds)) 
<choose_model_class>
def _resnet_split(m): return(m[0][6],m[1] )
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas = r_alphas))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas = alphas,
random_state = 42))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
l1_ratio=e_l1ratio))
lgbm_model = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11 ).fit(X_train,y_train)
 gbr_model = GradientBoostingRegressor().fit(X_train,y_train)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr_model,
lgbm_model),
meta_regressor=gbr_model,
use_features_in_secondary=True)
 stackX = np.array(X_train)
 stacky = np.array(y_train)
 stack_gen_model = stack_gen.fit(stackX, stacky )<predict_on_test>
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):
super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
self.logits = logits
self.reduce = reduce
def forward(self, inputs, targets):
if self.logits:
BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)
 else:
BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
 pt = torch.exp(-BCE_loss)
 F_loss = self.alpha *(1-pt)**self.gamma * BCE_loss
if self.reduce:
return torch.mean(F_loss)
 else:
return F_loss
em_preds = elastic_model.predict(X_test)
 lasso_preds = lasso_model.predict(X_test)
 ridge_preds = ridge_model.predict(X_test)
 stack_gen_preds = stack_gen_model.predict(X_test)
 lgbm_preds = lgbm_model.predict(X_test)
 gbr_preds = gbr_model.predict(X_test )<compute_test_metric>
learn = create_cnn(
 data,
resnet34,
cut=-2,
split_on=_resnet_split,
loss_func=FocalLoss(logits=True,alpha=alpha_log),
path=path,
metrics=[f1_score],
)
stack_preds =(( 0.1*em_preds)+(0.2*lasso_preds)+(0.1*ridge_preds)+(0.2 * gbr_preds)
 +(0.1*lgbm_preds)+(0.3*stack_gen_preds))
print('The accuracy of the stack Regressor is',r2_score(y_test,stack_preds))
print('RMSE is:
', mean_squared_error(y_test,stack_preds)) 
<concatenate>
learn.lr_find()
feats = test.select_dtypes(include=[np.number] ).interpolate().dropna()
feats = feats.drop([ 'Id'], axis=1 )<predict_on_test>
learn.fit_one_cycle(1,4e-2 )
em_preds = elastic_model.predict(feats)
 lasso_preds = lasso_model.predict(feats)
 ridge_preds = ridge_model.predict(feats)
 stack_gen_preds = stack_gen_model.predict(feats)
 lgbm_preds = lgbm_model.predict(feats)
 gbr_preds = gbr_model.predict(feats )<define_variables>
learn.unfreeze()
learn.lr_find(num_it=100 )
stack_preds =(( 0.1*em_preds)+(0.2*lasso_preds)+(0.1*ridge_preds)+(0.2 * gbr_preds)
 +(0.1*lgbm_preds)+(0.3*stack_gen_preds))<predict_on_test>
lr=1e-3
learn.fit_one_cycle(4, slice(lr/10,lr))
final_predictions = np.exp(stack_preds)
 print("Original predictions are:
", stack_preds[:5], "
")
 print("Final predictions are:
", final_predictions[:5] )<save_to_csv>
learn.save('4_epochs' )
submission = pd.DataFrame()
submission['Id'] = test.Id
submission['SalePrice'] = final_predictions
submission.to_csv('submission1.csv', index=False )<load_from_csv>
pred,y=learn.get_preds()
%matplotlib inline
titanic_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 warnings.filterwarnings(action='ignore', category=FutureWarning)
 titanic_data.info()
titanic_data.describe()
titanic_data.columns
titanic_data.shape 
<sort_values>
pred_test,_=learn.get_preds(ds_type=DatasetType.Test )
total = titanic_data.isnull().sum().sort_values(ascending=False)
 percent =(titanic_data.isnull().sum() /titanic_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head()<sort_values>
pred_test_tta,_=learn.TTA(ds_type=DatasetType.Test )
total = test_data.isnull().sum().sort_values(ascending=False)
 percent =(test_data.isnull().sum() /test_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head() 
<feature_engineering>
th_t = np.array([0.565,0.39,0.55,0.345,0.33,0.39,0.33,0.45,0.38,0.39,
0.34,0.42,0.31,0.38,0.49,0.50,0.38,0.43,0.46,0.40,
0.39,0.505,0.37,0.47,0.41,0.545,0.32,0.1] )
data["FamilySize"] = data['Parch'] + data['SibSp'] + 1
sns.barplot(x="FamilySize", y="Survived", data=data)
 data["IsAlone"] = 0
data["BigFamily"] = 0
data.loc[data['FamilySize'] == 1, 'IsAlone'] = 1
data.loc[data['FamilySize'] >= 5, 'BigFamily'] = 1<feature_engineering>
from sklearn.metrics import f1_score as f1_sc
ticket = pd.DataFrame(data['Ticket'].sort_values())
 ticket['SharedTicket'] = 0
ticket['SharedTicket'] =(ticket['Ticket'].eq(ticket['Ticket'].shift(1)) | ticket['Ticket'].eq(ticket['Ticket'].shift(-1)))
 ticket['SharedTicket'] = ticket['SharedTicket'].astype(int)
 data['SharedTicket'] = 0
data['SharedTicket'] = ticket['SharedTicket']
data.loc[data['SharedTicket']==1, 'Ticket'].sort_values()
data.loc[data['SharedTicket']==1, ['Ticket', 'Name']].sort_values("Ticket" ).head(20 )<categorify>
ths=np.arange(0.001,1,0.01)
 preds_s=F.sigmoid(pred)
 th_val=ths[np.argmax([f1_sc(y,preds_s>th,average='macro')for th in ths])]
def cabin_embarked(titanic_data):
df = titanic_data
df[['Cabin']] = df[['Cabin']].fillna(value="Unknown")
 df[['Embarked']] = df[['Embarked']].fillna(value="Unknown")
 titanic_data = df
return titanic_data
titanic_data = cabin_embarked(titanic_data)
 test_data = cabin_embarked(test_data)
 def shared_ticket(titanic_data):
ticket = pd.DataFrame(titanic_data['Ticket'].sort_values())
 ticket['SharedTicket'] = 0
ticket['SharedTicket'] =(ticket['Ticket'].eq(ticket['Ticket'].shift(1)) | ticket['Ticket'].eq(ticket['Ticket'].shift(-1)))
 ticket['SharedTicket'] = ticket['SharedTicket'].astype(int)
 titanic_data['SharedTicket'] = 0
titanic_data['SharedTicket'] = ticket['SharedTicket']
return titanic_data
titanic_data = shared_ticket(titanic_data)
 test_data = shared_ticket(test_data )<data_type_conversions>
eval_pred(pred,y )
def cabin_letters(titanic_data):
df = titanic_data
cabins = df.loc[df['Cabin'] != 'Unknown', 'Cabin'].str.split()
df['Cabin_numbers'] = cabins.transform(lambda x: len(x)).astype(int)
 df['Cabin_numbers'] = df['Cabin_numbers'].fillna(value=1)
 titanic_data['Cabin_letter'] = titanic_data['Cabin'].str[0]
titanic_data = df
return titanic_data
titanic_data = cabin_letters(titanic_data)
 test_data = cabin_letters(test_data )<feature_engineering>
def save_pred(pred, th=0.5, fname='protein_classification.csv'):
pred_list = []
for line in pred:
s = ' '.join(list([str(i)for i in np.nonzero(line>th)[0]]))
pred_list.append(s)
 sample_df = pd.read_csv(sample_csv)
 sample_list = list(sample_df.Id)
 pred_dic = dict(( key, value)for(key, value)
 in zip(test_ids,pred_list))
pred_list_cor = [pred_dic[id] for id in sample_list]
df = pd.DataFrame({'Id':sample_list,'Predicted':pred_list_cor})
 df.to_csv(fname, header=True, index=False )
def fill_unknown_cabins(titanic_data):
df = titanic_data
df.loc[(df['Pclass']==1)&(df['Cabin_letter']=='U'), 'Cabin_letter'] = 'B'
df.loc[(df['Pclass']==2)&(df['Cabin_letter']=='U'), 'Cabin_letter'] = 'D'
df.loc[(df['Pclass']==3)&(df['Cabin_letter']=='U'), 'Cabin_letter'] = 'G'
df.Cabin_letter.value_counts()
df.loc[df['Cabin_letter']=='T']
df.loc[df['Cabin_letter']=='T', ['Cabin_letter','Cabin']] = 'B'
titanic_data = df
return titanic_data
titanic_data = fill_unknown_cabins(titanic_data)
 test_data = fill_unknown_cabins(test_data)
 def family_size_alone(titanic_data):
titanic_data['FamilySize'] = titanic_data['SibSp'] + titanic_data['Parch'] + 1
titanic_data['IsAlone'] = 0
titanic_data['BigFamily'] = 0
titanic_data['Couple'] = 0
titanic_data.loc[titanic_data['FamilySize'] == 2, 'Couple'] = 1
titanic_data.loc[titanic_data['FamilySize'] == 1, 'IsAlone'] = 1
titanic_data.loc[titanic_data['FamilySize'] >= 5, 'BigFamily'] = 1
titanic_data = titanic_data.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 return titanic_data
titanic_data = family_size_alone(titanic_data)
 test_data = family_size_alone(test_data )<feature_engineering>
save_pred(to_np(F.sigmoid(pred_test)) , th=th_val, fname=f'protein_classification_{np.around(th_val,decimals=2)}.csv')
 save_pred(to_np(F.sigmoid(pred_test)) , th=th_t, fname='protein_classification_customth.csv' )
<feature_engineering><EOS> 
save_pred(to_np(F.sigmoid(pred_test_tta)) , th=th_val, fname=f'protein_classification_{np.around(th_val,decimals=2)}_tta.csv')
 save_pred(to_np(F.sigmoid(pred_test_tta)) , th=th_t, fname='protein_classification_customth_tta.csv' )