Kaggle competition: Severstal: Steel Defect Detection
class WarmupCosineSchedule(LambdaLR):

 def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):
self.warmup_steps = warmup_steps
self.t_total = t_total
self.cycles = cycles
self.name = 'Warmup Cosine'
super(WarmupCosineSchedule, self ).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)
 def lr_lambda(self, step):
if step < self.warmup_steps:
return float(step)/ float(max(1.0, self.warmup_steps))
progress = float(step - self.warmup_steps)/ float(max(1, self.t_total - self.warmup_steps))
return max(0.0, 0.5 *(1.+ math.cos(math.pi * float(self.cycles)* 2.0 * progress)))
 class WarmupCosineWithHardRestartsSchedule(LambdaLR):

 def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1):
self.warmup_steps = warmup_steps
self.t_total = t_total
self.cycles = cycles
self.name = 'Warmup Cosine With Hard Restarts'
super(WarmupCosineWithHardRestartsSchedule, self ).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)
 def lr_lambda(self, step):
if step < self.warmup_steps:
return float(step)/ float(max(1, self.warmup_steps))
progress = float(step - self.warmup_steps)/ float(max(1, self.t_total - self.warmup_steps))
if progress >= 1.0:
return 0.0
return max(0.0, 0.5 *(1.+ math.cos(math.pi *(( float(self.cycles)* progress)% 1.0)))) 
<find_best_params>
df_label = pd.read_csv('./resnet34-cls-tta-0.50_test.csv' ).fillna('')
 df_label[:16]
print(len(df_label))
!ls -la /kaggle/input/severstal-steel-defect-detection/test_images | wc 
(1804-3)*4
def get_num_params(model):
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
 n_params = sum([np.prod(p.size())for p in model_parameters])
 return n_params<find_best_params>
warnings.filterwarnings('ignore')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=NUM_EMBED, max_seq=MAX_SEQ,
enc_layers=NUM_LAYERS, heads=NUM_HEADS, dropout=0.1)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device);
n_params = get_num_params(model)
 print(f"Current model has {n_params} parameters." )<train_model>
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
def train_epoch(model, dataloader, optimizer, criterion, device="cuda"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
len_loader = len(dataloader)
 for idx, item in enumerate(dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optimizer.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optimizer.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc
def valid_epoch(model, valid_iterator, criterion, device="cuda"):
model.eval()
valid_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
len_dataset = len(valid_iterator)
 for idx, item in enumerate(valid_iterator):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
with torch.no_grad() :
output, _ = model(x, target_id)
 loss = criterion(output, label)
 valid_loss.append(loss.item())
 output = output[:, -1]
output = torch.sigmoid(output)
 label = label[:, -1]
pred =(output >= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(valid_loss)
 return loss, acc, auc<choose_model_class>
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )
optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE)
 scheduler = WarmupCosineWithHardRestartsSchedule(optimizer,
warmup_steps=WARMUP_EPOCHS,
cycles=NUM_RESTARTS,
t_total=EPOCHS)
 lr = []
best_auc = 0
best_epoch = 0
saving_threshold = 0.7675<train_model>
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
for epoch in range(EPOCHS):
loss, acc, auc = train_epoch(model,
dataloader,
optimizer,
criterion,
device)
 scheduler.step()
lr.append(optimizer.param_groups[0]['lr'])
 print(f"
Epoch - [{epoch+1}/{EPOCHS}]")
 print(f"Learning rate : {lr[-1]:.4e} ")
 print(f"train: loss - {loss:.3f} acc - {acc:.3f} auc - {auc:.3f}")
 val_loss, val_acc, val_auc = valid_epoch(model,
val_loader,
criterion,
device)
 print(f"valid: loss - {val_loss:.3f} acc - {val_acc:.3f} auc - {val_auc:.3f}")
 if best_auc < val_auc:
print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')
 best_auc = val_auc
best_epoch = epoch
if best_auc > saving_threshold:
model_name = f"sakt_layer_{NUM_LAYERS}_head_{NUM_HEADS}"
model_name += f"_embed_{NUM_EMBED}_seq_{MAX_SEQ}"
model_name += f"_auc_{val_auc:.4f}.pt"
torch.save(model.state_dict() , model_name )<drop_column>
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 } )
del dataset, valid_dataset<split>
df_mask = pd.DataFrame(res)
 df_mask = df_mask.fillna('')
 df_label_1 = pd.read_csv('./resnet34-cls-tta-0.50_test.csv' ).fillna('' ).sort_values(by=['ImageId_ClassId'])
 fp1 = list(df_label_1['EncodedPixels']=='')
 assert(np.all(df_mask['ImageId_ClassId'].values == df_label_1['ImageId_ClassId'].values))
print("How many predictions in the segementation result will be removed?")
 print(( df_mask.loc[fp1,'EncodedPixels'] != '' ).sum())
 print("How many postive predictions in the segementation?")
 print(( df_mask['EncodedPixels'] != '' ).sum())
 print("How many postive predictions in the classification?")
 print(( df_label_1['EncodedPixels'] != '' ).sum())
 df_mask.loc[fp1,'EncodedPixels']=''
df_mask.to_csv('submission.csv', index=False)
 df = df_mask
env = riiideducation.make_env()
iter_test = env.iter_test()<predict_on_test>
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()
%%time
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent < 95):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id',
'content_id',
'answered_correctly']]\ 
.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0],
prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1],
prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0,
['row_id', 'answered_correctly']] )<load_from_csv>
%matplotlib inline
df = pd.read_csv('submission.csv')[:40]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()