Kaggle competition: TGS Salt Identification Challenge
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<install_modules>
!pip3 install pycocotools
%%time
! python -W ignore.. /input/finalmoascripts/dae-book3b.py<load_from_zip>
%matplotlib inline
%reload_ext autoreload
%autoreload 2
print(torch.__version__)
 torch.cuda.is_available()
torch.backends.cudnn.benchmark=True
%%time
! python -W ignore.. /input/finalmoascripts/dae-book3c.py<install_modules>
MASKS_FN = 'train.csv'
TRAIN_DN = Path('train/images/')
 MASKS_DN = Path('train/masks/')
 TEST = Path('test/images/')
 PATH = Path('/kaggle/input/tgs-salt-identification-challenge/')
 PATH128 = Path('/tmp/128/')
 TMP = Path('/tmp/')
 MODEL = Path('/tmp/model/')
 PRETRAINED = Path('/kaggle/input/is-there-salt-resnet34/model/resnet34_issalt.h5')
 seg = pd.read_csv(PATH/MASKS_FN ).set_index('id')
 seg.head()
sz = 128
bs = 64
nw = 4
%%time
! python -W ignore.. /input/finalmoascripts/cv-inference-efficientnet.py<install_modules>
train_names_png = [TRAIN_DN/f for f in os.listdir(PATH/TRAIN_DN)]
train_names = list(seg.index.values)
 masks_names_png = [MASKS_DN/f for f in os.listdir(PATH/MASKS_DN)]
test_names_png = [TEST/f for f in os.listdir(PATH/TEST)]
%%time
! python -W ignore.. /input/finalmoascripts/cv-inference-efficientnet-augment.py<install_modules>
with ThreadPoolExecutor(4)as e: e.map(resize_mask, train_names_png )
%%time
! python -W ignore.. /input/finalmoascripts/inference_mx10.py<load_from_zip>
with ThreadPoolExecutor(4)as e: e.map(resize_mask, masks_names_png )
%%time
! python -W ignore.. /input/finalmoascripts/updated-inference-final-mx10-transfer-3.py<install_modules>
with ThreadPoolExecutor(4)as e: e.map(resize_mask, test_names_png )
%%time
! python -W ignore.. /input/finalmoascripts/inference_tabnet_dae.py<install_modules>
PATH = PATH128
%%time
! python -W ignore.. /input/finalmoascripts/inference-tabnet-1830.py<install_modules>
from datasets import CustomDataset
%%time
! python -W ignore.. /input/finalmoascripts/inference_pytorch_nn_01833.py<load_from_zip>
def dice(pred, targs):
pred =(pred>0 ).float()
return 2.*(pred*targs ).sum() /(pred+targs ).sum()
def IoU_np(pred, targs, thres=0):
pred =(pred>thres)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0)
 def IoU(pred, targs, thres=0):
pred =(pred>thres ).float()
intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )
%%time
! python -W ignore.. /input/finalmoascripts/NN1835.py<load_from_zip>
def get_base() :
layers = cut_model(f(True), cut)
 return nn.Sequential(*layers)
 def load_pretrained(model, path):
weights = torch.load(PRETRAINED, map_location=lambda storage, loc: storage)
 model.load_state_dict(weights, strict=False)
 return model
%%time
! python -W ignore.. /input/finalmoascripts/tabnet-public-184-inference.py<install_modules>
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
%%time
! python -W ignore.. /input/finalmoascripts/resnet-v64-infer.py<load_from_csv>
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv' ).set_index('sig_id')
 print(test.shape )<load_from_csv>
class UnetModel() :
def __init__(self,model,name='unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]
mx10 = pd.read_csv('submission_mx10.csv' ).set_index('sig_id')
 mx10_updated = pd.read_csv('submission_mx10_updated.csv' ).set_index('sig_id')
 tabnet_dae = pd.read_csv('submission_tabnet_01831.csv' ).set_index('sig_id')
 tabnet_1830 = pd.read_csv('submission_tabnet_01830.csv' ).set_index('sig_id')
 dae = pd.read_csv('submission_dae.csv' ).set_index('sig_id')
 dae_updated = pd.read_csv('submission_dae_updated.csv' ).set_index('sig_id')
 pytorch_nn = pd.read_csv('submission_pytorch_01833.csv' ).set_index('sig_id')
 keras_resnet = pd.read_csv('submission_RES_v64.csv' ).set_index('sig_id')
 effnet = pd.read_csv('submission_effnet.csv' ).set_index('sig_id')
 effnet_augment = pd.read_csv('submission_effnet_augment.csv' ).set_index('sig_id')
 tabnet_1840 = pd.read_csv('submission_tabnet_1840.csv' ).set_index('sig_id')
 NN_1835 = pd.read_csv('submission_NN1835.csv' ).set_index('sig_id' )<categorify>
x_names = [f'{x}.png' for x in train_names]
x_names_path = np.array([str(TRAIN_DN/x)for x in x_names])
 y_names = [x for x in x_names]
y_names_path = np.array([str(MASKS_DN/x)for x in x_names] )
df = pd.DataFrame()
df['TabNet_DAE'] = tabnet_dae.values.flatten()
df['TabNet_1830'] = tabnet_1830.values.flatten()
df['TabNet_1840'] = tabnet_1840.values.flatten()
df['MX10'] = mx10.values.flatten()
df['MX10_updated'] = mx10_updated.values.flatten()
df['PyTorch NN'] = pytorch_nn.values.flatten()
df['NN_1835'] = NN_1835.values.flatten()
df['Keras ResNet'] = keras_resnet.values.flatten()
df['DAE_0.2'] = dae.values.flatten()
df['DAE_0.3'] = dae_updated.values.flatten()
df['EffNet'] = effnet.values.flatten()
df['EffNet_augment'] = effnet_augment.values.flatten()<prepare_output>
aug_tfms = [RandomRotate(4, tfm_y=TfmType.CLASS),
RandomFlip(tfm_y=TfmType.CLASS),
RandomLighting(0.05, 0.05, tfm_y=TfmType.CLASS)]
sub =(( sub2 + sub7 + sub8)/3.+(sub1 + sub11)/2.+(sub3 + sub10)/2.+(sub4 + sub5 + sub9)/3.+(sub6 + sub12)/2.) /5.
 sub.reset_index(inplace=True )<normalization>
lr=3e-3
wd=1e-7
lrs = np.array([lr/100,lr/10,lr])
 n_folds = 8
out=np.zeros(( 18000,sz,sz))
alpha = 0
for i in range(n_folds):
val_size = 4000//n_folds
val_idxs=list(range(i*val_size,(i+1)*val_size)) 
(( val_x,trn_x),(val_y,trn_y)) = split_by_idx(val_idxs, x_names_path, y_names_path)
 test_x = np.array(test_names_png)
 tfms = tfms_from_model(resnet34, sz=sz, pad=0, crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)
 datasets = ImageData.get_ds(CustomDataset,(trn_x,trn_y),(val_x,val_y), tfms,(test_x, test_x), path=PATH)
 md = ImageData(PATH, datasets, bs=64, num_workers=nw, classes=None)
 denorm = md.trn_ds.denorm
f = resnet34
cut,lr_cut = model_meta[f]
m_base = load_pretrained(get_base() ,PRETRAINED)
 m = to_gpu(Unet34(m_base))
models = UnetModel(m)
 learn = ConvLearner(md, models, tmp_name=TMP, models_name=MODEL)
 learn.opt_fn=optim.Adam
learn.crit = lovasz_hinge
learn.metrics=[accuracy_thresh(0.5),dice, IoU]
learn.freeze_to(2)
 learn.fit(lr,2,wds=wd,cycle_len=10,use_clr_beta=(10,10, 0.85, 0.9))
learn.unfreeze()
learn.fit(lrs, 3, wds=wd, cycle_len=10,use_clr_beta=(10,10, 0.85, 0.9))
print(f'computing test set: {i}')
 out+=learn.predict(is_test=True)
 print('Computing optimal threshold')
 preds, targs = learn.predict_with_targs()
IoUs=[]
for a in np.arange(0, 1, 0.1):
IoUs.append(IoU_np(preds, targs, a))
IoU_max = np.array(IoUs ).argmax()
print(f'optimal Threshold: {IoU_max/10.0}')
 alpha+=IoU_max/10.0
def scale(x,k):
x = x.copy()
idx = np.where(x!=1)[0]
y = k * x[idx] /(1-x[idx])
 x[idx] = y/(1+y)
 return x<load_from_csv>
out = out/n_folds
alpha = alpha/n_folds
train = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 COLS = ['acetylcholine_receptor_agonist',
'acetylcholine_receptor_antagonist', 'adrenergic_receptor_agonist',
'adrenergic_receptor_antagonist',
'bacterial_cell_wall_synthesis_inhibitor',
'calcium_channel_blocker', 'cyclooxygenase_inhibitor',
'dna_inhibitor', 'dopamine_receptor_antagonist',
'estrogen_receptor_agonist', 'gaba_receptor_antagonist',
'glutamate_receptor_antagonist', 'histamine_receptor_antagonist',
'phosphodiesterase_inhibitor', 'serotonin_receptor_agonist',
'serotonin_receptor_antagonist', 'sodium_channel_inhibitor']<define_variables>
def rle_encode(im):

 pixels = im.flatten(order='F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
<normalization><EOS> 
sub = pd.DataFrame(list(zip(name_list, tmp_list)) , columns = ['id', 'rle_mask'] )