Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<choose_model_class>
from fastai.vision import *
import cv2
%%time
estimator=XGBRegressor(max_depth=10,
subsample=0.8,
colsample_bytree=0.9,
colsample_bylevel=0.7,
min_child_weight=200,
n_estimators=1000,
learning_rate=0.025,
objective="reg:squarederror",
tree_method="hist")
 estimator.fit(label_cat_not_num_train,y_train)
 importances=estimator.feature_importances_
predictions=estimator.predict(label_cat_not_num_test )<save_to_csv>
import mish_torch
from mish_torch import MishFunction,Mish
import bundled_gen_efficientnet
import gen_efficientnet
sample_submissions["item_cnt_month"]=predictions
sample_submissions.to_csv(f"submission.csv",index=False)
 sample_submissions.head()<set_options>
BS = 8
DROP_RATE = 0.0
STATS =(tensor([0.3436]), tensor([0.1961]))
INPUTS = Path('/kaggle/input')
 STATE = INPUTS/'severstal-eur-mish'
DATA = INPUTS/'severstal-steel-defect-detection'
CLASSES=['1','2','3','4']
THRESHOLDS = [0.29,0.552,0.447,0.330]
MIN_PXS = 2000
TTA_WEIGHTS = tensor([0.6, 0.2, 0.07, 0.07, 0.05] )
sns.set() 
<define_variables>
def mish_fn(x,inplace=False): return MishFunction.apply(x)
 class EfficientUnetEncoder(nn.Module):
def __init__(self, eff_net, first_skip=False):
super().__init__()
self.first_skip = first_skip
self.mod_names = ['conv_stem', 'bn1', 'act_fn', 'blocks']
for n in self.mod_names: setattr(self, n, getattr(eff_net, n))
self.blk_strides = [
max(( l.stride[0] if hasattr(l, 'stride')
 else 0
for l in flatten_model(blk)))
 for blk in self.blocks]
self.blk_channels = [
[ l.out_channels
for l in flatten_model(blk)
 if isinstance(l, nn.Conv2d)
 ][-1]
for blk in self.blocks]
def forward(self, x, hook=False):
x = self.conv_stem(x)
 x = self.bn1(x)
 x = self.act_fn(x)
 if hook:
hooks = []
for i,(blk,stride)in enumerate(zip(self.blocks,self.blk_strides)) :
if stride > 1 and(i > 0 or self.first_skip):
hooks.append(x)
 x = blk(x)
 else: x = self.blocks(x)
 return x if not hook else(x,hooks)
 def load_state(self, state_dict):
sd = {n: v for n,v in state_dict.items() if n.split('.')[0] in self.mod_names}
return self.load_state_dict(sd)
 class EfficientUnetClassifier(nn.Module):
def __init__(self, enc, eff_net, drop_rate=0.) :
super().__init__()
self.enc,self.drop_rate = enc,drop_rate
self.efficient_head,self.act_fn = eff_net.efficient_head, enc.act_fn
self.mod_names = ['conv_head','classifier']
if not self.efficient_head: self.mod_names.insert(1, 'bn2')
 for n in self.mod_names: setattr(self, n, getattr(eff_net, n))
def forward(self, x):
x = self.enc(x)
 if self.efficient_head:
x = F.adaptive_avg_pool2d(x, 1)
 x = self.conv_head(x)
 x = self.act_fn(x, inplace=True)
 else:
x = self.conv_head(x)
 x = self.bn2(x)
 x = self.act_fn(x, inplace=True)
 x = F.adaptive_avg_pool2d(x, 1)
 x = x.flatten(1)
 if self.drop_rate > 0.:
x = F.dropout(x, p=self.drop_rate, training=self.training)
 return self.classifier(x)
 return x
def load_state(self, state_dict):
res_enc = self.enc.load_state(state_dict)
 sd = {n: v for n,v in state_dict.items() if n.split('.')[0] in self.mod_names}
res = self.load_state_dict(sd)
 res.missing_keys.extend(res_enc.missing_keys)
 res.unexpected_keys.extend(res_enc.unexpected_keys)
 return res
class ResBlock(nn.Module):
def __init__(self, ni:int, nf:int, ks:int=3, stride:int=1, padding:int=None,
identity:bool=True, pre_activ:bool=True, bias:bool=True, act_fn:Callable[[None],nn.Module]=None,
leaky:float=None,init:Callable=nn.init.kaiming_normal_, norm_type:NormType=NormType.Batch):
super().__init__()
act = ifnone(act_fn, partial(relu, inplace=True, leaky=leaky))
if type(act)== type and issubclass(act, nn.Module): act = act()
if padding is None: padding =(ks-1)//2
if pre_activ: self.act1,self.bn1 = act,batchnorm_2d(ni, norm_type=norm_type)
 else: self.act1,self.bn1 = noop,noop
self.conv1 = conv2d(ni, nf, ks, stride=stride, padding=padding, bias=(bias and not pre_activ), init=init)
 self.bn2 = batchnorm_2d(nf, norm_type=norm_type)
 self.act2 = act
self.conv2 = conv2d(nf, nf, ks, padding=padding, bias=False, init=init)
 if identity:
l = [batchnorm_2d(ni, norm_type=norm_type)]
if ni != nf: l.append(conv2d(ni, nf, ks=1, bias=False, init=init))
if stride != 1: l.append(nn.AvgPool2d(2, ceil_mode=True))
self.identity = nn.Sequential(*l)
 else: self.identity = None
def forward(self, x):
res = self.act1(self.bn1(x))
res = self.conv1(res)
 res = self.act2(self.bn2(res))
res = self.conv2(res)
 if self.identity is not None: res += self.identity(x)
 return res
def gen_efficientnet(c_out=1000, c_in=1, act_fn=mish_fn, **kwargs):
channel_multiplier=1.0
depth_multiplier=1.0
arch_def = [
['ds_r1_k3_s2_e2_c16_se0.50'],
['ir_r2_k3_s1_e4_c24_se0.50'],
['ir_r2_k5_s2_e4_c40_se0.50'],
['ir_r3_k3_s1_e3_c80_se0.50'],
['ir_r3_k5_s2_e2_c112_se0.50'],
['ir_r3_k5_s1_e3_c160_se0.50'],
['ir_r1_k3_s2_e2_c192_se0.50'],
]
model = GenEfficientNet(
 decode_arch_def(arch_def, depth_multiplier),
num_classes=c_out,
in_chans=c_in,
stem_size=32,
channel_multiplier=channel_multiplier,
channel_divisor=8,
channel_min=None,
num_features=round_channels(1280, channel_multiplier, 8, None),
bn_args=_resolve_bn_args(kwargs),
act_fn=act_fn,
**kwargs 
)
 model.conv_stem.stride =(1,1)
 return model
class EfficientUResnet(nn.Module):
def __init__(self, c_out, c_in=1, drop_rate=0.0, resize_mode='nearest',
act_fn=mish_fn, res_act_fn=None, leaky=False, final_act=None, identity=True,
with_cfn=False, cfn_c_out=None, cfn_drop_rate=None, **kwargs):
super().__init__()
if type(act_fn)is type and issubclass(act_fn, nn.Module):
act = act_fn()
def _act_fn(self, x, inplace=False): return act(x)
 act_fn = _act_fn
resize_mode = {'mode': resize_mode}
if resize_mode == 'bilinear': resize_mode['align_corners'] = False
eff_net = gen_efficientnet(ifnone(cfn_c_out, c_out), c_in, act_fn=act_fn, **kwargs)
 self.enc = EfficientUnetEncoder(eff_net)
 if with_cfn:
self.classifier = EfficientUnetClassifier(self.enc, eff_net, drop_rate=ifnone(cfn_drop_rate, drop_rate))
self.resize = nn.Upsample(scale_factor=2, **resize_mode)
 res_args = dict(identity=identity, act_fn=res_act_fn, leaky=leaky)
 skip_nfs = [c for s,c in zip(self.enc.blk_strides[-1:1:-1], self.enc.blk_channels[-2::-1])
 if s == 2]
self.bridge = ResBlock(self.enc.blk_channels[-1], skip_nfs[0], ks=3, stride=1, **res_args)
 dec_nfs = skip_nfs[1:] + [self.enc.blk_channels[0]]
self.dec = nn.ModuleList([
ResBlock(ni*2, nf, **res_args)
 for ni,nf in zip(skip_nfs,dec_nfs)])
 self.final_conv = conv2d(dec_nfs[-1], c_out, ks=3, padding=1)
 self.final_act = final_act
def forward(self, inp):
x,skips = self.enc(inp, hook=True)
 x = self.bridge(x)
 for dec_blk,skip in zip(self.dec,reversed(skips)) :
x = self.resize(x)
 x = torch.cat(( x, skip), dim=1)
 x = dec_blk(x)
 x = self.resize(x)
 x = self.final_conv(x)
 if self.final_act: x = self.final_act(x)
 return x
def load_classifier_state(self, state_dict):
if 'model' in state_dict: state_dict = state_dict['model']
return self.enc.load_state(state_dict )
WARMUP_EPOCHS = 10
NUM_RESTARTS = 2
EPOCHS = 40
BATCH_SIZE = 256
VAL_BATCH_SIZE = 2048
MAX_SEQ = 160
NUM_HEADS = 8
NUM_EMBED = 128
NUM_LAYERS = 2
ACCEPTED_USER_CONTENT_SIZE = 3
n_skill = 13523
LEARNING_RATE = 5e-4<load_pretrained>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
%%time
dtypes = {'timestamp': 'int64',
'user_id': 'int32' ,
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8'}
train_cols = ['timestamp',
'user_id',
'content_id',
'content_type_id',
'answered_correctly']
train_df = pd.read_parquet('.. /input/cv-strategy-in-the-kaggle-environment/cv4_train.parquet')
 train_df = train_df[train_cols]
train_df = train_df.astype(dtypes)
 print(train_df.dtypes )<sort_values>
mdl = EfficientUResnet(c_out=4, act_fn=mish_fn, res_act_fn=Mish, drop_rate=DROP_RATE, with_cfn=False)
 state = torch.load(STATE/'seg.pth', map_location='cpu')
 if 'model' in state: state = state['model']
res = mdl.load_state_dict(state)
 mdl = mdl.cuda().eval()
res
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop=True)
 group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id')\ 
.apply(lambda r:(r['content_id'].values,
r['answered_correctly'].values))
del train_df
gc.collect() ;<load_from_csv>
df_samp_subm = pd.read_csv(DATA/'sample_submission.csv')
 df_samp_subm.head()
%%time
valid_df = pd.read_parquet('.. /input/cv-strategy-in-the-kaggle-environment/cv4_valid.parquet')
 valid_df = valid_df[train_cols]
valid_df = valid_df.astype(dtypes)
 valid_df = valid_df[valid_df.content_type_id == False]
val_group = valid_df[['user_id', 'content_id', 'answered_correctly']]\ 
.groupby('user_id')\ 
.apply(lambda r:(r['content_id'].values,
r['answered_correctly'].values))
del valid_df
gc.collect() ;<create_dataframe>
images = ImageId=df_samp_subm.ImageId_ClassId.str.slice(0, -2 ).unique()
df = pd.DataFrame.from_dict({'ImageId':images})
 df.head()
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = {}
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< ACCEPTED_USER_CONTENT_SIZE:
continue
if len(q)> self.max_seq:
total_questions = len(q)
 initial = total_questions % self.max_seq
if initial >= ACCEPTED_USER_CONTENT_SIZE:
self.user_ids.append(f"{user_id}_0")
 self.samples[f"{user_id}_0"] =(q[:initial], qa[:initial])
 for seq in range(total_questions // self.max_seq):
self.user_ids.append(f"{user_id}_{seq+1}")
 start = initial + seq * self.max_seq
end = initial +(seq + 1)* self.max_seq
self.samples[f"{user_id}_{seq+1}"] =(q[start:end], qa[start:end])
 else:
user_id = str(user_id)
 self.user_ids.append(user_id)
 self.samples[user_id] =(q, qa)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len == self.max_seq:
q[:] = q_
qa[:] = qa_
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label<create_dataframe>
data =(SegmentationItemList([], convert_mode='L', ignore_empty=True ).split_none().label_empty() 
.databunch(bs=BS, num_workers=2)
.normalize(STATS))
data.add_test(SegmentationItemList.from_df(df, path=DATA, folder='test_images', cols='ImageId', convert_mode='L'))
data
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=4)
 valid_dataset = SAKTDataset(val_group, n_skill)
 val_loader = DataLoader(valid_dataset,
batch_size=VAL_BATCH_SIZE,
shuffle=False,
drop_last=False,
num_workers=4)
 item = next(iter(dataloader))
val_item = next(iter(val_loader))
for i in range(len(item)) :
print(item[i].shape, val_item[i].shape )<choose_model_class>
thresh = torch.tensor(THRESHOLDS, dtype=torch.float32, device='cuda:0')[None,:,None,None]
tta_wgts = TTA_WEIGHTS.view(5, *([1]*4)).cuda()
batch_images = np.pad(images,(0,np.abs(len(images)% -BS)) , 'constant', constant_values='' ).reshape(-1, BS)
 def get_predictions(xb:Tensor):

 pred_raws = []
with torch.no_grad() : pred_raws.append(mdl(xb ).sigmoid())
 blks =(xb.view(*xb.shape[:-1], 4, -1)
.permute(0, 3, 1, 2, 4))
blks = blks.reshape(-1, *blks.shape[2:])
 for flip in range(4):
if flip == 1: blks = blks.flip(2)
 if flip == 2: blks = blks.flip(3)
 if flip == 3: blks = blks.flip(2)
 with torch.no_grad() : blk_raw = mdl(blks)
 if flip == 1: blk_raw = blk_raw.flip(2)
 if flip == 2: blk_raw = blk_raw.flip(( 2,3))
if flip == 3: blk_raw = blk_raw.flip(3)
 blk_raw =(blk_raw.view(-1, 4, *blk_raw.shape[1:])
.permute(0, 2, 3, 1, 4))
blk_raw = blk_raw.reshape(*blk_raw.shape[:-2], -1)
 pred_raws.append(blk_raw.sigmoid())
 pred_raw =(torch.stack(pred_raws)* tta_wgts ).sum(0)
 return(pred_raw > thresh ).to(dtype=torch.uint8, device='cpu' )
class FFN(nn.Module):
def __init__(self,
state_size=200,
bn_size=MAX_SEQ-1,
dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 self.bn = nn.BatchNorm1d(bn_size)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim,
heads = 8,
dropout = 0.2):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill,
max_seq=100,
embed_dim=128,
dropout = 0.2,
num_layers=1,
heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim)\
for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill,
max_seq=100,
embed_dim=128,
dropout = 0.2,
enc_layers=1,
heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill,
max_seq,
embed_dim,
dropout,
num_layers=enc_layers,
heads=heads)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<init_hyperparams>
df_subm = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df_subm.to_csv("submission.csv", index=False )