Kaggle competition: Severstal: Steel Defect Detection
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<merge>
!mkdir -p /tmp/pip/cache/
!cp.. /input/segmentation-models-zip-003/efficientnet_pytorch-0.4.0.xyz /tmp/pip/cache/efficientnet_pytorch-0.4.0.tar.gz
!cp.. /input/segmentation-models-zip-003/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp.. /input/segmentation-models-zip-003/segmentation_models_pytorch-0.0.3.xyz /tmp/pip/cache/segmentation_models_pytorch-0.0.3.tar.gz
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 test_df[TARGET] = model.predict(test_df[FEATS], num_iteration=model.best_iteration)
 set_predict(test_df[['row_id', TARGET]] )<load_from_csv>
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch
simulation = pd.read_csv('./submission.csv')
 simulation = simulation['answered_correctly']
simulation.head()<feature_engineering>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/senetunetmodelcode'
sys.path.append(package_path )
def bootstrap(x, repeats):
mean_x = np.average(x)
 sem_x = np.std(x)/np.sqrt(len(x))
median_x = np.median(x)
 vec = np.zeros(( 2, repeats))
for i in np.arange(repeats):
re = np.random.choice(len(x),len(x),replace=True)
 re_x = x[re]
vec[0,i] = np.mean(re_x)
 vec[1,i] = np.median(re_x)
 sns.distplot(vec[1,:], kde=True, rug=True, hist=True, norm_hist=True, axlabel='Frequency distribution')
 plt.show()
b_mean_sem = np.std(vec[0,:])
 b_median_sem = np.std(vec[1,:])
 return b_mean_sem, b_median_sem<compute_test_metric>
warnings.filterwarnings('ignore')
def Statistical_significance_test(A, B):
A = np.array(A)
 B = np.array(B)
 print('Paired t-test:', stats.ttest_rel(A, B))
A_var = np.var(A, ddof=1)
 B_var = np.var(B, ddof=1)
 A_df = len(A)- 1
B_df = len(B)- 1
f = A_var / B_var
one_sided_pval1 = stats.f.cdf(f, A_df, B_df)
 one_sided_pval2 = stats.f.sf(f, A_df, B_df)
 two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Student's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 3))
two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Welch's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 4))
print("Mann-Whitney's U-test:", stats.mannwhitneyu(A, B, alternative='two-sided'))
print("Wilcoxon signed rank sum test:", stats.wilcoxon(A, B))
return None<compute_test_metric>
unet_efficientnet = smp.Unet("efficientnet-b5", encoder_weights="imagenet", classes=4, activation="sigmoid" ).cuda()
ckpt_path = ".. /input/unet-efficientnet-baseline/unet_efficientnet_model.pth"
device = torch.device("cuda")
 unet_efficientnet.to(device)
 unet_efficientnet.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 unet_efficientnet.load_state_dict(state["state_dict"] )
b_mean_sem, b_median_sem = bootstrap(np.array(simulation), 1000000)
 Statistical_significance_test(A = simulation, B = np.repeat(b_mean_sem, len(simulation)) )<train_model>
ckpt_path = ".. /input/senetmodels/senext50_30_epochs_high_threshold.pth"
device = torch.device("cuda")
 senext50_30_epochs_high_threshold = Unet('se_resnext50_32x4d', encoder_weights=None, classes=4, activation=None ).cuda()
senext50_30_epochs_high_threshold.to(device)
 senext50_30_epochs_high_threshold.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 senext50_30_epochs_high_threshold.load_state_dict(state["state_dict"] )
print('Operation completed.' )<install_modules>
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
clf_resnet34 = load('/kaggle/input/severstalmodels/resnet34_classify.pth' ).cuda()
np.seterr(divide = 'ignore', invalid = 'ignore' )<define_variables>
ckpt_path = "/kaggle/input/pytorch-multi-label-classification/efficientnet_b0_model.pth"
device = torch.device("cuda")
 clf_efficientnet_b0 = EfficientNet.from_pretrained('efficientnet-b0', num_classes=4 ).cuda()
clf_efficientnet_b0.to(device)
 clf_efficientnet_b0.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 clf_efficientnet_b0.load_state_dict(state["state_dict"] )
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
ckpt_path = "/kaggle/input/clf-efficientnet-b4/efficientnet_b4_model.pth"
device = torch.device("cuda")
 clf_efficientnet_b4 = EfficientNet.from_pretrained('efficientnet-b4', num_classes = 4 ).cuda()
clf_efficientnet_b4.to(device)
 clf_efficientnet_b4.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 clf_efficientnet_b4.load_state_dict(state["state_dict"] )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns = set(data_types_dict.keys())).to_pandas()<data_type_conversions>
ckpt_path = "/kaggle/input/pytorch-multi-label-classification-effnet-b5/efficientnet_b5_model.pth"
device = torch.device("cuda")
 clf_efficientnet_b5 = EfficientNet.from_pretrained('efficientnet-b5', num_classes=4 ).cuda()
clf_efficientnet_b5.to(device)
 clf_efficientnet_b5.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 clf_efficientnet_b5.load_state_dict(state["state_dict"] )
train_df = train_df[train_df[target] != -1].reset_index(drop = True, inplace = False)
 train_df['prior_question_had_explanation'].fillna(False, inplace = True)
 train_df = train_df.astype(data_types_dict )<feature_engineering>
ckpt_path = "/kaggle/input/pytorch-multi-label-classification-effnet-b7/efficientnet_b7_model.pth"
device = torch.device("cuda")
 clf_efficientnet_b7 = EfficientNet.from_pretrained('efficientnet-b7', num_classes=4 ).cuda()
clf_efficientnet_b7.to(device)
 clf_efficientnet_b7.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 clf_efficientnet_b7.load_state_dict(state["state_dict"] )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns = ['lag'], inplace = True )<groupby>
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([
unet_se_resnext50_32x4d,
unet_mobilenet2,
unet_resnet34,
unet_efficientnet,
senext50_30_epochs_high_threshold
])
 model_clf = Model([
clf_efficientnet_b0,
clf_efficientnet_b4,
clf_efficientnet_b5,
clf_efficientnet_b7,
clf_resnet34
] )
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )<groupby>
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
train_df = train_df.groupby('user_id' ).tail(24 ).reset_index(drop = True )<load_from_csv>
def close(mask):

 kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(5,5))
closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
 return closing
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols = [0, 3],
dtype = {'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 train_df.drop(columns = ['question_id'], inplace = True )<data_type_conversions>
thresholds = [0.5, 0.75, 0.5, 0.75]
clf_thresholds = [0.5, 0.5, 0.75, 0.85]
min_area = [600, 1000, 1200, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
features_no_tta = None
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
if i == 0:
features_no_tta = features
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
clf_preds = torch.sigmoid(model_clf(features_no_tta)).detach().cpu().numpy()
for p, clf_pred, file in zip(preds, clf_preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 if clf_pred[i] <= clf_thresholds[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 else:
p_channel = close(p_channel)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 } )
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )<drop_column>
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )
valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace = True )<init_hyperparams>
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()
<train_model><EOS> 
%matplotlib inline
df = pd.read_csv('submission.csv')[:40]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()