Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<define_variables>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
return predictions
train_set = Pool(train_df[features], label = train_df[target])
 val_set = Pool(valid_df[features], label = valid_df[target] )<train_model>
!mkdir -p /tmp/pip/cache/
!cp /kaggle/input/segmentation-models-zip-003/efficientnet_pytorch-0.4.0.xyz /tmp/pip/cache/efficientnet_pytorch-0.4.0.tar.gz
!cp /kaggle/input/segmentation-models-zip-003/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp /kaggle/input/segmentation-models-zip-003/segmentation_models_pytorch-0.0.3.xyz /tmp/pip/cache/segmentation_models_pytorch-0.0.3.tar.gz
model = CatBoostClassifier(**params)
 model.fit(train_set, eval_set = val_set, use_best_model = True )<data_type_conversions>
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch
!pip install /kaggle/input/tta-pytorch/ttach-0.0.1-py3-none-any.whl
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<split>
warnings.filterwarnings('ignore')
try:
env = riiideducation.make_env()
except:
pass
iter_test = env.iter_test()
prior_test_df = None<prepare_output>
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop = True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 user_sum = np.zeros(len(test_df), dtype = np.int16)
 user_count = np.zeros(len(test_df), dtype = np.int16)
 content_sum = np.zeros(len(test_df), dtype = np.int32)
 content_count = np.zeros(len(test_df), dtype = np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_correctness'] = user_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df[target] = np.average([
model.predict_proba(test_df[features].values)[:,1],
model1.predict(test_df[features])
 ], weights=[0.49934,0.50066], axis=0)
 env.predict(test_df[['row_id', target]] )<import_modules>
def flip(x, dim):
indices = [slice(None)] * x.dim()
indices[dim] = torch.arange(x.size(dim)- 1, -1, -1,
dtype=torch.long, device=x.device)
 return x[tuple(indices)]
class ChannelTranspose(ImageOnlyTransform):
def get_transform_init_args_names(self):
return()
def get_params_dependent_on_targets(self, params):
pass
def __init__(self, axes=(2, 0, 1)) :
super().__init__(always_apply=True)
 self.axes = axes
def apply(self, img, **params):
return np.transpose(img, self.axes)
 class ImageDataset(Dataset):
def __init__(
 self,
*,
img_folder: str,
fold_csv: str = None,
fold_number: int = None,
is_test: bool = False,
gray_scale: bool = False,
num_classes=2,
max_count=None,
meta_cols=() ,
transforms=None,
postprocess_func: Callable[[Dict], Dict] = None,
include_image_orig=False 
):
self.img_folder = img_folder
if fold_csv:
df = pd.read_csv(fold_csv)
 if fold_number is not None:
if is_test:
self.data = df[df['fold'] == fold_number]
else:
self.data = df[df['fold'] != fold_number]
else:
self.data = df
else:
self.data = pd.DataFrame(
 {'image': os.listdir(img_folder)} ).sort_values(by='image')
 self.data = self.data.to_dict(orient='row')
 if max_count is not None:
self.apply_max_count(max_count)
 for row in self.data:
self.preprocess_row(row)
 self.transforms = transforms
self.gray_scale = gray_scale
self.num_classes = num_classes
self.meta_cols = meta_cols
self.postprocess_func = postprocess_func
self.include_image_orig = include_image_orig
def apply_max_count(self, max_count):
if isinstance(max_count, Number):
self.data = self.data[:max_count]
else:
data = defaultdict(list)
 for row in self.data:
data[row['label']].append(row)
 min_index = np.argmin(max_count)
 min_count = len(data[min_index])
 for k, v in data.items() :
count = int(min_count *(max_count[k] / max_count[min_index]))
data[k] = data[k][:count]
self.data = [v for i in range(len(data)) for v in data[i]]
def preprocess_row(self, row: dict):
row['image'] = os.path.join(self.img_folder, row['image'])
 def __len__(self):
return len(self.data)
 def _get_item_before_transform(self, row: dict, item: dict):
pass
def _get_item_after_transform(self, row: dict,
transformed: dict,
res: dict):
if 'label' in row:
res['targets'] = ast.literal_eval(str(row['label']))
if isinstance(res['targets'], list):
res['targets'] = np.array(res['targets'], dtype=np.float32)
 def __getitem__(self, index):
row = self.data[index]
image = self.read_image_file(row['image'], self.gray_scale)
 item = {'image': image}
self._get_item_before_transform(row, item)
 if self.transforms:
item = self.transforms(**item)
 if self.gray_scale:
item['image'] = np.expand_dims(item['image'], axis=0)
 res = {
'features': item['image'].astype(np.float32),
'image_file': row['image']
}
if self.include_image_orig:
res['image'] = image
for c in self.meta_cols:
res[c] = row[c]
self._get_item_after_transform(row, item, res)
 if self.postprocess_func:
res = self.postprocess_func(res)
 return res
@staticmethod
def read_image_file(path: str, gray_scale=False):
if path.endswith('.tiff')and not gray_scale:
return tifffile.imread(path)
 elif path.endswith('.npy'):
return np.load(path)
 else:
if gray_scale:
img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
 assert img is not None, \
f'Image at path {path} does not exist'
return img.astype(np.uint8)
 else:
img = cv2.imread(path)
 assert img is not None, \
f'Image at path {path} does not exist'
return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 class TtaWrap(Dataset):
def __init__(self, dataset: Dataset, tfms=()):
self.dataset = dataset
self.tfms = tfms
def __getitem__(self, item):
return self.dataset[item]
def __len__(self):
return len(self.dataset)
 def inverse(self, a: np.array):
last_dim = len(a.shape)- 1
for t in self.tfms:
if isinstance(t, albu.HorizontalFlip):
a = flip(a, last_dim)
 elif isinstance(t, albu.VerticalFlip):
a = flip(a, last_dim - 1)
 elif isinstance(t, albu.Transpose):
axis =(0, 1, 3, 2)if len(a.shape)== 4 else(0, 2, 1)
 a = a.permute(*axis)
 return a
import numpy as np
import pandas as pd
import category_encoders as ce
import gc
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing, metrics<load_from_csv>
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
train = dt.fread(".. /input/riiid-test-answer-prediction/train.csv" ).to_pandas()<load_from_csv>
def create_transforms(additional):
res = list(additional)
 res.extend([
albu.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = albu.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[albu.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
<data_type_conversions>
thresholds = [0.50, 0.50, 0.50, 0.50]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel = post_process(p_channel, thresholds[i], min_area[i])
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df.to_csv('submission.csv', index=False)
%%time
data_types_dict = {
'row_id': 'int64',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'
}
train = train.astype(data_types_dict)
 train['prior_question_had_explanation'].fillna(False, inplace=True )<set_options>
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )
gc.collect()<drop_column>
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()