Kaggle competition: Severstal: Steel Defect Detection
test["sub_best"] = test['.. /input/eda-modelling-of-the-external-data-inc-ensemble/external_meta_ensembled']
col_comment = ["id", "image_name", "patient_id", "sub_best"]
col_diff = [column for column in test.columns if "diff" in column]
test_diff = test[col_comment + col_diff].reset_index(drop=True)
 test_diff["diff_avg"] = test_diff[col_diff].mean(axis=1 )<feature_engineering>
!pip install.. /input/pretrainedmodels3/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript3'
sys.path.append(package_path )
test_diff["sub_new"] = test_diff.apply(lambda x:(1+WEIGHT*x["diff_avg"])*x["sub_best"] if x["diff_avg"]<0 else(1-WEIGHT*x["diff_avg"])*x["sub_best"] + WEIGHT*x["diff_avg"] , axis=1 )<feature_engineering>
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F
from model import Unet
submission["target"] = sub_best["target"]
submission.head()<feature_engineering>
model_classification = Resnet34_classification()
model_classification.load_state_dict(torch.load('.. /input/clsification3/00007500_model.pth', map_location=lambda storage, loc: storage), strict=True )
submission.loc[test["id"], "target"] = test_diff["sub_new"].values<save_to_csv>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
submission.to_csv("submission.csv", index=False)
 submission.head()<define_variables>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
DEVICE = "TPU"
BASEPATH = ".. /input/siim-isic-melanoma-classification"<install_modules>
batch_size = 1
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225 )
!pip install -q efficientnet<import_modules>
df = pd.read_csv(sample_submission_path )
random.seed(a=128)
<train_on_grid>
testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=0,
pin_memory=True
)
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print("REPLICAS: ", strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE 
<define_variables>
threshold_label = [0.50,0.50,0.50,0.50,]
EPOCHS = 6
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [384, 384]<load_from_csv>
augment = ['null']
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 TRAINING_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
TEST_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))
CLASSES = [0,1]<categorify>
probability_label, image_id = get_classification_preds(model_classification, testset)
 predict_label = probability_label>np.array(threshold_label ).reshape(1,4,1,1)
 image_id_class_id = []
encoded_pixel = []
for b in range(len(image_id)) :
for c in range(4):
image_id_class_id.append(image_id[b]+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle ='1 1'
encoded_pixel.append(rle)
 df_classification = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'] )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
ckpt_path = ".. /input/res18fullsize3/model_swa.pth"
device = torch.device("cuda")
 model_segmentation = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model_segmentation.to(device)
 model_segmentation.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model_segmentation.load_state_dict(state["state_dict"] )
def lrfn(epoch):
LR_START = 0.000005
LR_MAX = 0.000020 * strategy.num_replicas_in_sync
LR_MIN = 0.000001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr 
<init_hyperparams>
models = [model_segmentation]
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, BatchNormalization, Activation, Dropout
from tensorflow.keras.regularizers import l2 
<choose_model_class>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
with strategy.scope() :
model7 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 model7.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
modelRN = tf.keras.Sequential([
tf.keras.applications.ResNet152V2(
 input_shape=(331, 331, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 modelRN.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 with strategy.scope() :
modelIV3 = tf.keras.Sequential([
tf.keras.applications.InceptionV3(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
GlobalAveragePooling2D() ,
Dense(1, activation='sigmoid')
 ])
 modelIV3.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy']
)<train_model>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 history7 = model7.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 historyRN = modelRN.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule])
 historyIV3 = modelIV3.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS, callbacks=[lr_schedule] )<predict_on_test>
threshold_pixel = [0.5,0.5,0.5,0.5,]
min_size = [200,1500,1500,2000]
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilitiesRN = modelRN.predict(test_images_ds)
 probabilities7 = model7.predict(test_images_ds)
 probabilitiesV3 = modelIV3.predict(test_images_ds )<save_to_csv>
def null_augment(input): return input
def flip_lr_augment(input): return torch.flip(input, dims=[2])
 def flip_ud_augment(input): return torch.flip(input, dims=[3])
 def null_inverse_augment(logit): return logit
def flip_lr_inverse_augment(logit): return torch.flip(logit, dims=[2])
 def flip_ud_inverse_augment(logit): return torch.flip(logit, dims=[3])
 augment =(
(null_augment, null_inverse_augment), 
(flip_lr_augment,flip_lr_inverse_augment), 
(flip_ud_augment,flip_ud_inverse_augment),
)
print('Generating submission.csv files...')
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )<prepare_output>
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
images = images.cuda()
batch_preds = 0
probabilities = []
for model in models:
model = model.cuda()
for k,(a, inv_a)in enumerate(augment):
logit = model(a(images))
p = inv_a(torch.sigmoid(logit))
if k ==0:
probability = p**0.5
else:
probability += p**0.5
probability = probability/len(augment)
 probabilities.append(probability)
 batch_preds+=probability
batch_preds = batch_preds.data.cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, threshold_pixel[cls], min_size[cls])
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df_segmentation = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'] )
pred_dfRN = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilitiesRN)})
 pred_df7 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities7)})
 pred_dfV3 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilitiesV3)})
 pred_df7.head()<prepare_output>
df= df_segmentation.copy()
subRN = sub.copy()
sub7 = sub.copy()
subV3 = sub.copy()
subRN.head()<save_to_csv>
if 1:
df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))
del subRN['target']
subRN = subRN.merge(pred_dfRN, on='image_name')
 subRN.to_csv('submissionRN.csv', index=False)
 del subV3['target']
subV3 = subV3.merge(pred_dfV3, on='image_name')
 subV3.to_csv('submissionV3.csv', index=False)
 del sub7['target']
sub7 = sub7.merge(pred_df7, on='image_name')
 sub7.to_csv('submission7.csv', index=False)
 subRN.head()<save_to_csv>
df_mask = df_segmentation.copy()
df_label = df_classification.copy()
ensemble1 =(subRN['target'] + sub7['target']+ subV3['target'])/3
ensemble_img1 = subRN['image_name']
ensemble_sub1 = pd.concat([ensemble_img1, ensemble1], axis = 1)
 ensemble_sub1.to_csv('submissionV3.csv', index=False)
 ensemble_sub1.head()<import_modules>
assert(np.all(df_mask['ImageId_ClassId'].values == df_label['ImageId_ClassId'].values))
print(( df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels'] != '' ).sum())
 df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels']=''
import numpy as np
import pandas as pd
from scipy.stats import rankdata
import os
import re<load_from_csv>
df_mask.to_csv("submission.csv", index=False )
def Stacking(input_folder,
best_base,
output_path,
column_names,
cutoff_lo,
cutoff_hi):

 sub_base = pd.read_csv(best_base)
 all_files = os.listdir(input_folder)
 nb_files = len(all_files)
 assert type(input_folder)== str, "Wrong type"
assert type(best_base)== str, "Wrong type"
assert type(output_path)== str, "Wrong type"
assert type(cutoff_lo)in [float, int], "Wrong type"
assert type(cutoff_hi)in [float, int], "Wrong type"
assert(cutoff_lo >= 0)&(cutoff_lo <= 1)&(cutoff_hi >= 0)&(cutoff_hi <= 1), "cutoff_lo and cutoff_hi must be between 0 and 1"
assert len(column_names)== 2, "Only two columns must be in column_names"
assert type(column_names[0])== str, "Wrong type"
assert type(column_names[1])== str, "Wrong type"
concat_sub = pd.DataFrame()
concat_sub[column_names[0]] = sub_base[column_names[0]]
for index, f in enumerate(all_files):
concat_sub[column_names[1]+str(index)] = pd.read_csv(input_folder + f)[column_names[1]]
print(" ***** 1/4 Read and concatenate submissions SUCCESSFUL *****")
 concat_sub['target_max'] = concat_sub.iloc[:, 1:].max(axis=1)
 concat_sub['target_min'] = concat_sub.iloc[:, 1:].min(axis=1)
 concat_sub['target_mean'] = concat_sub.iloc[:, 1:].mean(axis=1)
 concat_sub['target_median'] = concat_sub.iloc[:, 1:].median(axis=1)
 print(" ***** 2/4 Get the data fields ready for stacking SUCCESSFUL *****")
 concat_sub['target_base'] = sub_base[column_names[1]]
concat_sub[column_names[1]] = np.where(np.all(concat_sub.iloc[:, 1:] > cutoff_lo, axis=1),
concat_sub['target_max'],
np.where(np.all(concat_sub.iloc[:, 1:] < cutoff_hi, axis=1),
concat_sub['target_min'],
concat_sub['target_base']))
print(" ***** 3/4 Set up cutoff threshold for lower and upper bounds SUCCESSFUL *****")
 concat_sub[column_names].to_csv(output_path, index=False, float_format='%.12f')
 print(" ***** 4/4 Generating Stacked dataframe SUCCESSFUL *****")
 print(" ***** COMPLETED *****" )<define_variables>
if 1:
df_mask['Class'] = df_mask['ImageId_ClassId'].str[-1].astype(np.int32)
 df_mask['Label'] =(df_mask['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df_mask['Class']==1)&(df_mask['Label']==1)).sum()
pos2 =(( df_mask['Class']==2)&(df_mask['Label']==1)).sum()
pos3 =(( df_mask['Class']==3)&(df_mask['Label']==1)).sum()
pos4 =(( df_mask['Class']==4)&(df_mask['Label']==1)).sum()
num_image = len(df_mask)//4
num = len(df_mask)
 pos =(df_mask['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))