Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<train_model>
model = CatBoostClassifier(**params)
 model.fit(train_set, eval_set = val_set, use_best_model = True )<data_type_conversions>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
max_timestamp_u_dict = max_timestamp_u.set_index('user_id' ).to_dict()<split>
try:
env = riiideducation.make_env()
except:
pass
iter_test = env.iter_test()
prior_test_df = None<feature_engineering>
import os
import cv2
import torch
import pandas as pd
import numpy as np
import glob
import gc
from tqdm import tqdm
import matplotlib.pyplot as plt
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Resize, Compose)
 from albumentations.pytorch.transforms import ToTensor
import torch.utils.data as data
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop = True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 user_sum = np.zeros(len(test_df), dtype = np.int16)
 user_count = np.zeros(len(test_df), dtype = np.int16)
 content_sum = np.zeros(len(test_df), dtype = np.int32)
 content_count = np.zeros(len(test_df), dtype = np.int32)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id, content_id,timestamp)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values,test_df['timestamp'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i] = timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i] =(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
test_df['user_correctness'] = user_sum / user_count
test_df['Accuracy_sum'] = user_sum
test_df['Questions_num'] = user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df["lagtime"] = lagtime
test_df["lagtime_mean"] = lagtime_mean
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 test_df[target] = model.predict_proba(test_df[features])[:,1]
env.predict(test_df[['row_id', target]] )<import_modules>
class SteelDataset(Dataset):
def __init__(self, df, augment=None):
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
def __len__(self):
return len(self.fnames)
 def __getitem__(self, index):
image_id = self.fnames[index]
image = cv2.imread(test_data_folder + '/%s'%(image_id), cv2.IMREAD_COLOR)
 return image_id, image
<import_modules>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
import gc
import psutil
import joblib
import random
from tqdm import tqdm
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader<define_variables>
def null_collate(batch):
batch_size = len(batch)
 input = []
infor = []
for b in range(batch_size):
input.append(batch[b][1])
 infor.append(batch[b][0])
 input = np.stack(input ).astype(np.float32)/255
input = input.transpose(0,3,1,2)
 input = torch.from_numpy(input ).float()
return infor, input
TRAIN_SAMPLES = 320000
MAX_SEQ = 100
MIN_SAMPLES = 5
EMBED_DIM = 128
DROPOUT_RATE = 0.2
LEARNING_RATE = 1e-3
MAX_LEARNING_RATE = 2e-3
EPOCHS = 30
TRAIN_BATCH_SIZE = 2048<data_type_conversions>
df = pd.read_csv(sample_submission_path)
 test_dataset = SteelDataset(df)
 test_loader = DataLoader(
 test_dataset,
batch_size = 2,
drop_last = False,
num_workers = 0,
pin_memory = True,
collate_fn = null_collate
)
%%time
dtypes = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}
train_df = pd.read_feather('.. /input/riiid-cross-validation-dataset/train.feather')[[
'timestamp', 'user_id', 'content_id', 'content_type_id', 'answered_correctly'
]]
for col, dtype in dtypes.items() :
train_df[col] = train_df[col].astype(dtype)
 train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True)
 train_df.reset_index(drop=True, inplace=True )<count_unique_values>
def null_augment(input): return input
def flip_lr_augment(input): return torch.flip(input, dims=[2])
 def flip_ud_augment(input): return torch.flip(input, dims=[3])
 def null_inverse_augment(logit): return logit
def flip_lr_inverse_augment(logit): return torch.flip(logit, dims=[2])
 def flip_ud_inverse_augment(logit): return torch.flip(logit, dims=[3])
 augment =(
(null_augment, null_inverse_augment), 
(flip_lr_augment,flip_lr_inverse_augment), 
(flip_ud_augment,flip_ud_inverse_augment),
)
skills = train_df["content_id"].unique()
joblib.dump(skills, "skills.pkl.zip")
 n_skill = len(skills)
 print("number skills", len(skills))<load_pretrained>
TEMPERATE=0.5
def probability_mask_to_probability_label(probability):
batch_size,num_class,H,W = probability.shape
probability = probability.permute(0, 2, 3, 1 ).contiguous().view(batch_size,-1, 5)
 value, index = probability.max(1)
 probability = value[:,1:]
return probability
def remove_small_one(predict, min_size):
H,W = predict.shape
num_component, component = cv2.connectedComponents(predict.astype(np.uint8))
predict = np.zeros(( H,W), np.bool)
 for c in range(1,num_component):
p =(component==c)
 if p.sum() >min_size:
predict[p] = True
return predict
def remove_small(predict, min_size):
for b in range(len(predict)) :
for c in range(4):
predict[b,c] = remove_small_one(predict[b,c], min_size[c])
 return predict
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
joblib.dump(group, "group.pkl.zip")
 del train_df
gc.collect()<filter>
def do_evaluate_segmentation(net, test_loader, augment=[]):
def sharpen(p,t=TEMPERATE):
if t!=0:
return p**t
else:
return p
test_num = 0
test_id = []
test_probability_label = []
test_probability_mask = []
test_truth_label = []
test_truth_mask = []
for t,(fnames, input)in enumerate(tqdm(test_loader)) :
batch_size,C,H,W = input.shape
input = input.cuda()
with torch.no_grad() :
net.eval()
num_augment = 0
if 1:
logit = net(input)
 probability = torch.softmax(logit,1)
 probability_mask = sharpen(probability,0)
 num_augment+=1
if 'flip_lr' in augment:
logit = net(torch.flip(input,dims=[3]))
probability = torch.softmax(torch.flip(logit,dims=[3]),1)
 probability_mask += sharpen(probability)
 num_augment+=1
if 'flip_ud' in augment:
logit = net(torch.flip(input,dims=[2]))
probability = torch.softmax(torch.flip(logit,dims=[2]),1)
 probability_mask += sharpen(probability)
 num_augment+=1
probability_mask = probability_mask/num_augment
probability_label = probability_mask_to_probability_label(probability_mask)
 probability_mask =(probability_mask.data.cpu().numpy() *255 ).astype(np.uint8)
 probability_label =(probability_label.data.cpu().numpy() *255 ).astype(np.uint8)
 test_id.extend([i for i in fnames])
 test_probability_mask.append(probability_mask)
 test_probability_label.append(probability_label)
 test_probability_mask = np.concatenate(test_probability_mask)
 test_probability_label = np.concatenate(test_probability_label)
 return test_probability_label, test_probability_mask, test_id
train_indexes = list(group.index)[:TRAIN_SAMPLES]
valid_indexes = list(group.index)[TRAIN_SAMPLES:]
train_group = group[group.index.isin(train_indexes)]
valid_group = group[group.index.isin(valid_indexes)]
del group, train_indexes, valid_indexes
print(len(train_group), len(valid_group))<define_variables>
ckpt_file = '.. /input/henge5/trace_model_swa.pth'
net = torch.jit.load(ckpt_file ).cuda()
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, min_samples=1, max_seq=128):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = {}
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< min_samples:
continue
if len(q)> self.max_seq:
total_questions = len(q)
 initial = total_questions % self.max_seq
if initial >= min_samples:
self.user_ids.append(f"{user_id}_0")
 self.samples[f"{user_id}_0"] =(q[:initial], qa[:initial])
 for seq in range(total_questions // self.max_seq):
self.user_ids.append(f"{user_id}_{seq+1}")
 start = initial + seq * self.max_seq
end = start + self.max_seq
self.samples[f"{user_id}_{seq+1}"] =(q[start:end], qa[start:end])
 else:
user_id = str(user_id)
 self.user_ids.append(user_id)
 self.samples[user_id] =(q, qa)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len == self.max_seq:
q[:] = q_
qa[:] = qa_
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label<create_dataframe>
probability_label, probability_mask, image_id = do_evaluate_segmentation(net, test_loader, augment=['null'] )
train_dataset = SAKTDataset(train_group, n_skill, min_samples=MIN_SAMPLES, max_seq=MAX_SEQ)
 train_dataloader = DataLoader(train_dataset, batch_size=TRAIN_BATCH_SIZE, shuffle=True, num_workers=8)
 valid_dataset = SAKTDataset(valid_group, n_skill, max_seq=MAX_SEQ)
 valid_dataloader = DataLoader(valid_dataset, batch_size=TRAIN_BATCH_SIZE, shuffle=False, num_workers=8 )<define_search_model>
del net
gc.collect()
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=128, embed_dim=128, dropout_rate=0.2):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=dropout_rate)
 self.dropout = nn.Dropout(dropout_rate)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<train_model>
threshold_label = [ 0.70, 0.8, 0.50, 0.70,]
threshold_mask_pixel = [ 0.6, 0.8, 0.5, 0.6,]
threshold_mask_size = [ 1, 1, 1, 1,]
def train_fn(model, dataloader, optimizer, scheduler, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
for item in dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
target_mask =(target_id != 0)
 optimizer.zero_grad()
output, _, = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optimizer.step()
scheduler.step()
train_loss.append(loss.item())
 output = torch.masked_select(output, target_mask)
 label = torch.masked_select(label, target_mask)
 pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc<categorify>
predict_label = probability_label>(np.array(threshold_label)*255 ).astype(np.uint8 ).reshape(1,4)
 predict_mask = probability_mask>(np.array(threshold_mask_pixel)*255 ).astype(np.uint8 ).reshape(1,4,1,1 )
def valid_fn(model, dataloader, criterion, device="cpu"):
model.eval()
valid_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
for item in dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
target_mask =(target_id != 0)
 output, _, = model(x, target_id)
 loss = criterion(output, label)
 valid_loss.append(loss.item())
 output = torch.masked_select(output, target_mask)
 label = torch.masked_select(label, target_mask)
 pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(valid_loss)
 return loss, acc, auc<choose_model_class>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_DIM, dropout_rate=DROPOUT_RATE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE)
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.OneCycleLR(
 optimizer, max_lr=MAX_LEARNING_RATE, steps_per_epoch=len(train_dataloader), epochs=EPOCHS 
)
 model.to(device)
 criterion.to(device )<train_model>
image_id_class_id = []
encoded_pixel = []
for b in range(len(image_id)) :
for c in range(4):
image_id_class_id.append(image_id[b]+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle = mask2rle(predict_mask[b,c])
 encoded_pixel.append(rle )
best_auc = 0
max_steps = 3
step = 0
for epoch in range(EPOCHS):
loss, acc, auc = train_fn(model, train_dataloader, optimizer, scheduler, criterion, device)
 print("epoch - {}/{} train: - {:.3f} acc - {:.3f} auc - {:.3f}".format(epoch+1, EPOCHS, loss, acc, auc))
loss, acc, auc = valid_fn(model, valid_dataloader, criterion, device)
 print("epoch - {}/{} valid: - {:.3f} acc - {:.3f} auc - {:.3f}".format(epoch+1, EPOCHS, loss, acc, auc))
if auc > best_auc:
best_auc = auc
step = 0
torch.save(model.state_dict() , "sakt_model.pt")
 else:
step += 1
if step >= max_steps:
break<drop_column>
df = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv('submission.csv', index=False )
del train_dataset, valid_dataset<save_model>
def summarise_submission_csv(df):
text = ''
df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
neg1 = num_image-pos1
neg2 = num_image-pos2
neg3 = num_image-pos3
neg4 = num_image-pos4
text += 'compare with LB probing...
 '
text += '\t\tnum_image = %5d(1801)
 '%num_image
text += '\t\tnum = %5d(7204)
 '%num
text += '
'
text += '\t\tpos1 = %5d(128)%0.3f
'%(pos1,pos1/128)
 text += '\t\tpos2 = %5d(43)%0.3f
'%(pos2,pos2/43)
 text += '\t\tpos3 = %5d(741)%0.3f
'%(pos3,pos3/741)
 text += '\t\tpos4 = %5d(120)%0.3f
'%(pos4,pos4/120)
 text += '
'
text += '\t\tneg1 = %5d(1673)%0.3f %3d
'%(neg1,neg1/1673, neg1-1673)
 text += '\t\tneg2 = %5d(1758)%0.3f %3d
'%(neg2,neg2/1758, neg2-1758)
 text += '\t\tneg3 = %5d(1060)%0.3f %3d
'%(neg3,neg3/1060, neg3-1060)
 text += '\t\tneg4 = %5d(1681)%0.3f %3d
'%(neg4,neg4/1681, neg4-1681)
 text += '--------------------------------------------------
'
text += '\t\tneg = %5d(6172)%0.3f %3d
'%(neg,neg/6172, neg-6172)
 text += '
'
if 1:
pass
return text
torch.save(model.state_dict() , "sakt_model_final.pt" )<split>
def rle2mask(mask_rle, shape=(1600,256)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
<load_pretrained><EOS> 
%matplotlib inline
df = pd.read_csv('submission.csv')[:60]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(test_data_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()