Kaggle competition: Severstal: Steel Defect Detection
train_df = train_df.groupby('user_id' ).tail(500 ).reset_index(drop = True )<load_from_csv>
warnings.filterwarnings('ignore')
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols = [0, 3],
dtype = {'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 train_df.drop(columns = ['question_id'], inplace = True)
<categorify>
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )<data_type_conversions>
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift() 
<feature_engineering>
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16')
<categorify>
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df.to_csv('submission.csv', index=False)
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg = explanation_agg.astype('int16')
<data_type_conversions>
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
 lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])
 train_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])
 train_df.lagtime_mean=train_df.lagtime_mean.astype('int32')
 train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
<drop_column>
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()
<define_variables><EOS> 
%matplotlib inline
df = pd.read_csv('submission.csv')[:40]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()