Kaggle competition: Severstal: Steel Defect Detection
sub = pd.read_csv('.. /working/submission.csv')
 sub['answered_correctly'].hist(bins=15 )<install_modules>
warnings.filterwarnings('ignore')
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
unet_se_resnext50_32x4d = load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
np.seterr(divide = 'ignore', invalid = 'ignore' )<define_variables>
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )
data_types_dict = {
'user_id': 'int32',
'timestamp': 'int64',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns = set(data_types_dict.keys())).to_pandas() 
<train_model>
thresholds = [0.5, 0.7, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df = df.fillna('' )
print('Training dataset detailed information')
 print('*' * 50)
 print('Columns:', train_df.columns)
 print('*' * 50)
 print('Shape:', train_df.shape)
 print('*' * 50)
 print('NA values in each column:', sum(train_df.isna().sum()))
print('*' * 50 )<data_type_conversions>
import matplotlib.pyplot as plt
import random
import math as math
import os
import numpy as np
import pandas as pd
import cv2
from PIL import Image
import os
import tensorflow as tf
import keras.backend as K
from keras.utils import Sequence
from keras.applications import Xception
from keras.layers import UpSampling2D, Conv2D, Activation, LeakyReLU, BatchNormalization
from keras import Model
from keras.losses import binary_crossentropy
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
from keras.optimizers import Adam
import imgaug as ia
import imgaug.augmenters as iaa
import segmentation_models as sm
import efficientnet.keras as efn
from keras.optimizers import Adam
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
train_df = train_df[train_df[target] != -1].reset_index(drop = True, inplace = False)
 train_df['prior_question_had_explanation'].fillna(False, inplace = True)
 train_df = train_df.astype(data_types_dict)
<feature_engineering>
base = efn.EfficientNetB3(weights=None, include_top=False, input_shape=(256, 1600, 3), pooling='avg')
 base.trainable=True
dropout_dense_layer = 0.2
classifier_model = Sequential()
classifier_model.add(base)
 classifier_model.add(Dropout(dropout_dense_layer))
classifier_model.add(Dense(4, activation='sigmoid'))
classifier_model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] 
)
 classifier_model.load_weights('.. /input/single-models/B3_opt_multi_new3.h5')
 IMAGE_RGB_MEAN = [0.485, 0.456, 0.406]
IMAGE_RGB_STD = [0.229, 0.224, 0.225]
def normalize(images):
images = images / 255.0
images[:,:,:,0] =(images[:,:,:,0]-IMAGE_RGB_MEAN[0])/IMAGE_RGB_STD[0]
images[:,:,:,1] =(images[:,:,:,1]-IMAGE_RGB_MEAN[1])/IMAGE_RGB_STD[1]
images[:,:,:,2] =(images[:,:,:,2]-IMAGE_RGB_MEAN[2])/IMAGE_RGB_STD[2]
return images
def denormalize(images):
images[:,:,:,0] = images[:,:,:,0]*IMAGE_RGB_STD[0]+IMAGE_RGB_MEAN[0]
images[:,:,:,1] = images[:,:,:,1]*IMAGE_RGB_STD[1]+IMAGE_RGB_MEAN[1]
images[:,:,:,2] = images[:,:,:,2]*IMAGE_RGB_STD[2]+IMAGE_RGB_MEAN[2]
images = np.array(images * 255 ).astype('uint8')
 TEST_PATH = '.. /input/severstal-steel-defect-detection/test_images/'
def prepareData(source, path, cleanup=True):
source['defect'] = False
if cleanup: source.EncodedPixels = ''
source['ClassId'] = source['ImageId_ClassId'].str[-1:]
source['ImageId'] = source['ImageId_ClassId'].str[:-2]
source = source[['ImageId_ClassId', 'ImageId','ClassId','defect','EncodedPixels']]
source.ClassId = source.ClassId.astype('int')
 source['path'] = path + source['ImageId']
return source
prior_question_elapsed_time_mean = train_df.prior_question_elapsed_time.dropna().values.mean()
train_df['prior_question_elapsed_time_mean'] = train_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean )<groupby>
test_df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 test_df = prepareData(test_df, TEST_PATH)
 ids = test_df['ImageId'].unique()
test_df.EncodedPixels = ''
class_weights = [0.5, 0.5, 0.5, 0.5]
for picIdx in tqdm_notebook(range(len(ids))):
batch = np.zeros(( 4, 256, 1600, 3))
filename = ids[picIdx]
img = cv2.imread(TEST_PATH+filename)
 batch[0, :, :, :] = img
batch[1, :, :, :] = img[:,::-1,:]
batch[2, :, :, :] = img[::-1,:,:]
batch[3, :, :, :] = img[::-1,::-1,:]
batch = normalize(batch)
 classTTA = classifier_model.predict(batch)
 hasDefect = np.mean(classTTA, axis=0)>class_weights
for i, classId in enumerate(hasDefect):
if classId == False:
name = filename+"_"+str(i+1)
 line = test_df[test_df.ImageId_ClassId == name].index[0]
test_df.loc[line, 'EncodedPixels'] = -1
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns = ['lag'], inplace = True)
<groupby>
mark = test_df[test_df.EncodedPixels == -1]
mark.ClassId.value_counts()
sub = df.copy()
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )<categorify>
for filename in mark.ImageId_ClassId:
sub.loc[sub.ImageId_ClassId == filename, 'EncodedPixels'] = -1
sub = sub.replace(-1, '')
 sub.to_csv('submission.csv', index=False)
 sub.head(20 )
train_df['Accuracy_sum'] = train_df['user_id'].map(user_agg['sum'])
 train_df['Questions_num'] = train_df['user_id'].map(user_agg['count'] )<groupby>
sub['Image'] = sub['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 sub['Class'] = sub['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 sub['empty'] = sub['EncodedPixels'].map(lambda x: not x)
 sub[sub['empty'] == False]['Class'].value_counts()