Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<prepare_x_and_y>
%matplotlib inline
def get_score(santa_path, numb_list):
sum_dist = 0
for i in range(len(santa_path)- 1):
city1 = santa_path[i]
city2 = santa_path[i+1]
if city1 > city2:
city1, city2 = city2, city1
x1 = city_X[city1]
y1 = city_Y[city1]
x2 = city_X[city2]
y2 = city_Y[city2]
city1_dict = dist_dict[city1]
if city2 in city1_dict:
dist = city1_dict[city2]
else:
dist = get_dist(x1, y1, x2, y2)
 dist_dict[city1][city2] = dist
if(( numb_list[i] + 1)% 10 == 0)and(not santa_path[i] in plist):
dist *= 1.1
sum_dist += dist
return sum_dist<define_variables>
PadIfNeeded,
HorizontalFlip,
VerticalFlip,
CenterCrop,
Crop,
Compose,
Transpose,
RandomRotate90,
ElasticTransform,
GridDistortion,
OpticalDistortion,
RandomSizedCrop,
OneOf,
CLAHE,
RandomBrightnessContrast,
RandomGamma,
RandomContrast,
InvertImg,
RandomContrast
)
best_path = subm['Path'].tolist()<define_variables>
os.listdir('/kaggle/input/severstal-steel-defect-detection')
 train_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/train.csv')
 train_df['Image_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[0])
 train_df['class_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[1])
 train_df['mask']= ~train_df[train_df.columns[1]].isna()
print(train_df.shape)
 train_df.head(5 )
dist_dict = {i:{} for i in range(cities_numb)}<compute_test_metric>
mask_count=train_df.groupby(by='Image_id' ).agg(np.sum ).reset_index()
print(mask_count.shape)
 mask_count.head(3 )
def pop(cur_perm, numb_list, score):
city1 = cur_perm[0]
city2 = cur_perm[1]
x1 = city_X[city1]
y1 = city_Y[city1]
x2 = city_X[city2]
y2 = city_Y[city2]
dist = get_dist(x1, y1, x2, y2)
 pos = numb_list[0]
if(( pos + 1)% 10 == 0 and(not city1 in plist)) :
return cur_perm[1:], numb_list[1:], score - dist * 1.1
else:
return cur_perm[1:], numb_list[1:], score - dist
def push(cur_perm, numb_list, score, x):
city1 = cur_perm[-1]
city2 = x
x1 = city_X[city1]
y1 = city_Y[city1]
x2 = city_X[city2]
y2 = city_Y[city2]
dist = get_dist(x1, y1, x2, y2)
 pos = numb_list[-1] + 1
if(( pos)% 10 == 0 and(not city1 in plist)) :
return cur_perm + [x], numb_list + [pos], score + dist * 1.1
else:
return cur_perm + [x], numb_list + [pos], score + dist<categorify>
mask_count.sort_values(by='mask',ascending=False,inplace=True)
 mask_count.head()
def improve_perm(santa_path, len_perm):
santa_len = len(santa_path)
 L = 150000
R = len_perm + L
numb_list = list(range(L, R))
cur_perm = santa_path[L:R]
cur_dist = get_score(cur_perm, numb_list)
 for i in tqdm(range(0, santa_len - R - 1)) :
if i % 15 == 0:
for file_name in filenames:
new_path = []
indx = sorted([sub_perm_dict[file_name][x] for x in cur_perm[1:-1]])
 for ind in indx:
new_path += [path_dict[file_name][int(ind)]]
cur_path = [cur_perm[0]] + new_path + [cur_perm[-1]]
cur_path_rev = [cur_perm[0]] + new_path[::-1] + [cur_perm[-1]]
cur_score = get_score(cur_path, numb_list)
 if(cur_score < cur_dist and cur_dist - cur_score > 0.05):
print('GO ', cur_score, cur_dist, get_score(cur_perm, numb_list))
ans = list(santa_path)
 ans[L:R] = cur_path
return ans
cur_score = get_score(cur_path_rev, numb_list)
 if(cur_score < cur_dist and cur_dist - cur_score > 0.05):
print('REV ', cur_score, cur_dist)
 ans = list(santa_path)
 ans[L:R] = cur_path_rev
return ans
cur_perm, numb_list, cur_dist = pop(cur_perm, numb_list, cur_dist)
 cur_perm, numb_list, cur_dist = push(cur_perm, numb_list, cur_dist, santa_path[R])
 L += 1
R += 1<feature_engineering>
sub_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id']=sub_df[sub_df.columns[0]].apply(lambda x : x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() ,columns=['Image_id'])
 print(test_imgs.shape)
 test_imgs.head()
best_path2 = improve_perm(best_path, 100 )<compute_test_metric>
train_path = '/kaggle/input/severstal-steel-defect-detection/train_images/'
train_dir=os.listdir(train_path)
 train_dir[:3]
test_path = '/kaggle/input/severstal-steel-defect-detection/test_images/'
test_dir = os.listdir(test_path )
get_score(best_path, list(range(len(best_path))))<compute_test_metric>
def mask2rle(img):
pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape):
depth = len(rles)
 masks = np.zeros(( *input_shape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle, input_shape)
 return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles
get_score(best_path2, list(range(len(best_path2))))<create_dataframe>
pa='/kaggle/input/efficientnet'
sys.path.append(os.path.abspath('/kaggle/input/efficientnet-models/efficientnet-master/'))
N_CLASSES=4
def ClsModel_2(n_classes=1, input_shape=(254,600,3)) :
base_model = EfficientNetB2(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(5,5), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1600, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model = ClsModel_2(n_classes=N_CLASSES)
 N_CLASSES=4
def ClsModel_3(n_classes=1, input_shape=(254,254,3)) :
base_model = EfficientNetB3(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(3,3), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1024, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model_class = ClsModel_3(n_classes=N_CLASSES)
 N_CLASSES=4
def ClsModel_1(n_classes=1, input_shape=(128,800,3)) :
base_model = EfficientNetB1(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(3,3), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1024, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model_class_b1 = ClsModel_1(n_classes=N_CLASSES)
ans = pd.DataFrame()
ans['Path'] = best_path2<save_to_csv>
class DataGenerator1(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(600,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
ans.to_csv('submission.csv', index = None )<import_modules>
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 254), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(254,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
import numpy as np
import pandas as pd
import numba
from sympy import isprime, primerange
from math import sqrt
from sklearn.neighbors import KDTree
from tqdm import tqdm_notebook as tqdm
from itertools import combinations, permutations
from functools import lru_cache<load_from_csv>
class DataGenerator3(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(128, 800), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(800,128))
if self.augmentaion[0]==1:
img_i[i,] = img
continue;
aug_no = np.random.randint(0, len(self.augmentaion))
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<compute_test_metric>
all_classifer_path=['/kaggle/input/efficent-b2-600/EfficientNetB3_efficenent_254_600_exp_4.h5',
'/kaggle/input/efficient977/EfficientNetB3_efficenent_exp_2.h5',
'/kaggle/input/efficient-b1-9793/EfficientNetB1_efficenent_128_800_exp_1.h5']
class_models =[model,model_class,model_class_b1]
pred_class_i=[]
pred=[]
DataGenerator=[DataGenerator1,DataGenerator2,DataGenerator3]
for i in range(2,3):
augmentaion=[1,HorizontalFlip,VerticalFlip]
for aug in augmentaion:
class_models[i].load_weights(all_classifer_path[i])
 test_generator = DataGenerator[i](
 test_imgs.index,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
augmentaion=[aug],
batch_size=1,
n_classes=4 
)
 batch_pred_class = class_models[i].predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 pred.append(batch_pred_class.round().astype(int))
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path)
 @numba.jit
def chunk_scores(chunk):
scores = np.zeros(10)
 pure_distance = 0
for i in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[i], chunk[i+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if is_not_prime[id_from]:
scores[9-i%10] += distance
scores *= 0.1
scores += pure_distance
return scores<compute_train_metric>
test=pred.copy()
@numba.jit('f8(i8, i8, i8[:], i8[:], i8[:], i8, f8[:,:], i8[:])', nopython=True, parallel=False)
 def score_compound_chunk(offset, head, firsts, lasts, lens, tail, scores, indexes):
score = 0.0
last_city_id = head
for i in numba.prange(len(indexes)) :
index = indexes[i]
first, last, chunk_len = firsts[index], lasts[index], lens[index]
score += cities_distance(offset, last_city_id, first)
 score += scores[index,(offset + 1)% 10]
last_city_id = last
offset += chunk_len
return score + cities_distance(offset, last_city_id, tail)
 @numba.jit('i8(i8, i8, i8[:], i8[:], i8[:], i8, f8[:,:], i8[:,:], f8)', nopython=True, parallel=False)
 def best_score_permutation_index(offset, head, firsts, lasts, lens, tail, scores, indexes, best_score):
best_index = -1
for i in numba.prange(len(indexes)) :
score = score_compound_chunk(offset, head, firsts, lasts, lens, tail, scores, indexes[i])
 if score < best_score:
best_index, best_score = i, score
return best_index<statistical_test>
test_1=np.array(test)
 np.sum(test_1[0],axis=0 )
kdt = KDTree(XY)
 fives = set()
for i in tqdm(cities.index):
dists, neibs = kdt.query([XY[i]], 9)
 for comb in combinations(neibs[0], 5):
if all(comb):
fives.add(tuple(sorted(comb)))
 neibs = kdt.query_radius([XY[i]], 10, count_only=False, return_distance=False)
 for comb in combinations(neibs[0], 5):
if all(comb):
fives.add(tuple(sorted(comb)))
 print(f'{len(fives)} cities fives are selected.')
 @numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def sum_distance(ids):
res = 0
for i in numba.prange(len(ids)) :
for j in numba.prange(i + 1, len(ids)) :
res += cities_distance(0, ids[i], ids[j])
 return res
fives = np.array(list(fives))
distances = np.array(list(map(sum_distance, tqdm(fives))))
order = distances.argsort()
fives = fives[order]<load_from_csv>
test_1=np.array(test)
 test_1=test_1[0]+test_1[1]+test_1[2]
test_1[np.where(test_1<=1)]=0
test_1[np.where(test_1>1)]=1
np.sum(test_1,axis=0 )
path = np.array(pd.read_csv('.. /input/dp-shuffle/submission.csv' ).Path )<statistical_test>
np.sum(test_1.round().astype(int),axis=0 )
@lru_cache(maxsize=None)
 def indexes_permutations(n):
return np.array(list(map(list, permutations(range(n)))))
 path_index = np.argsort(path[:-1])
 print(f'Total score is {score_path(path):.2f}.')
 for _ in range(2):
for ids in tqdm(fives[:2 * 10**6]):
i1, i2, i3, i4, i5 = np.sort(path_index[ids])
 head, tail = path[i1-1], path[i5+1]
chunks = [path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3],
path[i3:i3+1], path[i3+1:i4], path[i4:i4+1], path[i4+1:i5], path[i5:i5+1]]
chunks = [chunk for chunk in chunks if len(chunk)]
scores = np.array([chunk_scores(chunk)for chunk in chunks])
 lens = np.array([len(chunk)for chunk in chunks])
 firsts = np.array([chunk[0] for chunk in chunks])
 lasts = np.array([chunk[-1] for chunk in chunks])
 best_score = score_compound_chunk(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) [0])
 index = best_score_permutation_index(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) , best_score)
 if index > 0:
perm = [chunks[i] for i in indexes_permutations(len(chunks)) [index]]
path[i1-1:i5+2] = np.concatenate([[head], np.concatenate(perm), [tail]])
 path_index = np.argsort(path[:-1])
 print(f'New total score is {score_path(path):.3f}.Permutating path at indexes {i1}, {i2}, {i3}, {i4}, {i5}.' )<save_to_csv>
aaa=[128/43/741/120]
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False)
 make_submission(score_path(path), path )<set_options>
np.sum(batch_pred_class )
%matplotlib inline<load_from_csv>
pred_class =test_1.round().astype(int )
original = pd.read_csv(".. /input/close-ends-chunks-optimization-aka-2-opt/1515651.377838111.csv")
 cities = pd.read_csv(".. /input/traveling-santa-2018-prime-paths/cities.csv")
 cities.rename(columns={"CityId":"Path"}, inplace=True)
 baseline = original.merge(cities,how='left',on='Path' )<feature_engineering>
np.sum(pred_class,axis=0 )
def score_it(df):
df['step'] = np.sqrt(( df.X - df.X.shift())**2 +(df.Y - df.Y.shift())**2)
 df['step_adj'] = np.where(( df.index)% 10 != 0, df.step, df.step +
df.step*0.1*(~df.Path.shift().isin(pnums)))
 return df.step_adj.sum()
display(score_it(baseline))<normalization>
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,
batch_size=1, dim=(256, 1600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
img_path = f"{self.base_path}/{im_name}"
img =cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
 img_i[i,]=img
return img_i
def __generate_y(self, ID):
img_i = np.empty(( self.batch_size,*self.dim,self.n_classes),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
img =build_masks(rles, input_shape=self.dim)
 img_i[i,]=img
return img_i
def __check_input__(self,index):
X,y=self.__getitem__(index)
 indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
ID=list_IDs_batch[0]
X=X[0]
y=y[0]
im_name=self.df['Image_id'].loc[ID]
img= cv2.imread(train_path+im_name)
 image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 fig,axis = plt.subplots(nrows=1,ncols=1, figsize=(20, 20))
for i in range(1):
try:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis[i].imshow(img_i)
 except IndexError:
print("IndexError : so skip")
 img_i=mask_image(X,y)
 axis.imshow(img_i,cmap='gray')
 break
except TypeError:
img_i=mask_image(X,y)
 print("i")
 axis.imshow(img_i,cmap='gray')
 print("TypeError(axis[i] is fails): so skip")
 break
img_i=mask_image(img.astype(np.uint8),masks.astype(np.uint8))
plt.figure(figsize=(10,10))
plt.imshow(img_i )
def create_mat(df):
mat = pdist(df)
 return squareform(mat)
 def create_distance_callback(dist_matrix):
def distance_callback(from_node, to_node):
return int(dist_matrix[from_node][to_node])
 return distance_callback
status_dict = {0: 'ROUTING_NOT_SOLVED',
1: 'ROUTING_SUCCESS',
2: 'ROUTING_FAIL',
3: 'ROUTING_FAIL_TIMEOUT',
4: 'ROUTING_INVALID'}
def optimize(df, startnode, stopnode, fixed):
num_nodes = df.shape[0]
mat = create_mat(df)
 dist_callback = create_distance_callback(mat)
 search_parameters = pywrapcp.RoutingModel.DefaultSearchParameters()
search_parameters.solution_limit = num_iters
search_parameters.first_solution_strategy =(
 routing_enums_pb2.FirstSolutionStrategy.LOCAL_CHEAPEST_INSERTION)
 search_parameters.local_search_metaheuristic =(
 routing_enums_pb2.LocalSearchMetaheuristic.GUIDED_LOCAL_SEARCH)
 if fixed:
routemodel = pywrapcp.RoutingModel(num_nodes, 1, [startnode], [stopnode])
 else:
routemodel = pywrapcp.RoutingModel(num_nodes, 1, startnode)
 routemodel.SetArcCostEvaluatorOfAllVehicles(dist_callback)
 assignment = routemodel.SolveWithParameters(search_parameters)
 return routemodel, assignment
def get_route(df, startnode, stopnode, fixed):
routemodel, assignment = optimize(df, int(startnode), int(stopnode), fixed)
 route_number = 0
node = routemodel.Start(route_number)
 route = []
while not routemodel.IsEnd(node):
route.append(node)
 node = assignment.Value(routemodel.NextVar(node))
return route<define_variables>
non_missing_train_idx = mask_count[mask_count['mask'] > 0]
def run_opt(df,m,n):
i = 0
while i < n:
startpoint = random.randint(0,df.shape[0])
 endpoint = min(( startpoint + m),df.shape[0])
 district = df.iloc[startpoint:endpoint,:3].copy()
district = district.reset_index()
locations = district[['X', 'Y']].values
segnodes = get_route(locations, 0,(m-1), fixed=True)
 ord_district = district.iloc[segnodes]
segment = ord_district.index.tolist()
temp = district.loc[segment, ['Path','X', 'Y']].reset_index()
district_2 = district.copy()
district_2.iloc[:(m-1),1:] = temp.copy()
district = district.set_index('index')
 district_2 = district_2.set_index('index')
 district['step'] = np.sqrt(( district.X - district.X.shift())**2 +(district.Y - district.Y.shift())**2)
 district['step_adj'] = np.where(( district.index)% 10 != 0, district.step, district.step +
district.step*0.1*(~district.Path.shift().isin(pnums)))
 district_2['step'] = np.sqrt(( district_2.X - district_2.X.shift())**2 +(district_2.Y - district_2.Y.shift())**2)
 district_2['step_adj'] = np.where(( district_2.index)% 10 != 0, district_2.step, district_2.step +
district_2.step*0.1*(~district_2.Path.shift().isin(pnums)))
 check_dist = district.step_adj.sum() > district_2.step_adj.sum()
print(i)
 print(district.step_adj.sum() , district_2.step_adj.sum())
 if check_dist:
df.iloc[startpoint:endpoint,0:3] = district_2
i += 1<feature_engineering>
BATCH_SIZE = 10
train_idx, val_idx = train_test_split(
 non_missing_train_idx.index,
random_state=25,
test_size=0.2 
)
 train_generator = DataGenerator(
 train_idx,
df=non_missing_train_idx,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4,
mode='fit' 
)
 val_generator = DataGenerator(
 val_idx,
df=non_missing_train_idx,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4 
)
 train_generator.__check_input__(0 )
def score_it(df):
df['step'] = np.sqrt(( df.X - df.X.shift())**2 +(df.Y - df.Y.shift())**2)
 df['step_adj'] = np.where(( df.index)% 10 != 0, df.step, df.step +
df.step*0.1*(~df.Path.shift().isin(pnums)))
 return df.step_adj.sum()
score = score_it(baseline)
 display(score )<save_to_csv>
def dice_coef(y_true, y_pred, smooth= 1e-6):
y_pred = K.round(y_pred)
 intersection = K.sum(y_true * y_pred, axis=(1,2))
return K.mean(( 2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth))
def dice_loss(y_true, y_pred,smooth = 1e-9):
intersection = K.sum(y_true * y_pred, axis=(1,2))
score =(2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth)
 return K.mean(1-score)
 def bce_dice_loss(y_true, y_pred):
return dice_loss(y_true, y_pred )
sub = pd.read_csv(".. /input/traveling-santa-2018-prime-paths/sample_submission.csv")
 sub['Path'] = baseline['Path']
sub.to_csv('submission_{}.csv'.format(score), index=False)
 sub.head()<import_modules>
def dice_coef(y_true, y_pred, smooth= 1e-6):
y_pred = K.round(y_pred)
 intersection = K.sum(y_true * y_pred,axis=(1,2))
return K.mean(( 2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth))
opt=keras.optimizers.Adam(lr=1e-4)
import numpy as np
import pandas as pd
import numba
from sympy import isprime, primerange
from math import sqrt
from sklearn.neighbors import KDTree
from tqdm import tqdm
from itertools import combinations, permutations
from functools import lru_cache<load_from_csv>
model_seg = load_model('/kaggle/input/seg-model/seg_model.h5')
 model_seg.load_weights('/kaggle/input/16000-model/1600.h5' )
K = 4
NUM_NEIGHBORS = 5
RADIUS = 0
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<compute_test_metric>
np.sum(pred_class,axis=0 )
@numba.jit('f8(i8, i8)', nopython=True, parallel=False)
 def euclidean_distance(id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
return sqrt(dx * dx + dy * dy)
 @numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):

 distance = euclidean_distance(id_from, id_to)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):

 pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
distance = euclidean_distance(id_from, id_to)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path)
 @numba.jit
def chunk_scores(chunk):
scores = np.zeros(10)
 pure_distance = 0
for i in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[i], chunk[i+1]
distance = euclidean_distance(id_from, id_to)
 pure_distance += distance
if is_not_prime[id_from]:
scores[9-i%10] += distance
scores *= 0.1
scores += pure_distance
return scores
def score_compound_chunk(offset, head, chunks, tail, scores, indexes_permutation=None):

 if indexes_permutation is None:
indexes_permutation = range(len(chunks))
score = 0.0
last_city_id = head
for index in indexes_permutation:
chunk, chunk_scores = chunks[index], scores[index]
score += cities_distance(offset % 10, last_city_id, chunk[0])
 score += chunk_scores[(offset + 1)% 10]
last_city_id = chunk[-1]
offset += len(chunk)
 return score + cities_distance(offset % 10, last_city_id, tail )<load_from_csv>
sub_df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id'] = sub_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() , columns=['Image_id'])
 test_imgs=test_imgs
filtered_mask = sub_df['Image_id'].isin(test_imgs["Image_id"].values)
 filtered_sub_df = sub_df[filtered_mask].copy()
null_sub_df = sub_df[~filtered_mask].copy()
null_sub_df['EncodedPixels'] = null_sub_df['EncodedPixels'].apply(
 lambda x: ' ')
 filtered_sub_df.reset_index(drop=True, inplace=True)
 test_imgs.reset_index(drop=True, inplace=True)
 print(filtered_sub_df.shape)
 print(null_sub_df.shape)
 filtered_sub_df.head()
path = pd.read_csv('.. /input/santatour1515602/1515602.86080513.csv', squeeze=True ).values
initial_score = score_path(path)
 print("Initial tour distance(score): {:.2f}".format(initial_score))<randomize_order>
all_class=pred_class.flatten()
all_class
filtered_sub_df['EncodedPixels']=all_class
def not_trivial_permutations(iterable):
perms = permutations(iterable)
 next(perms)
 yield from perms
@lru_cache(maxsize=None)
 def not_trivial_indexes_permutations(length):
return np.array([list(p)for p in not_trivial_permutations(range(length)) ])
 path_index = np.argsort(path[:-1])
 for _ in range(1):
for ids in tqdm(candidates):
idx = sorted(path_index[ids])
 head, tail = path[idx[0] - 1], path[idx[-1] + 1]
chunks = [path[idx[0]:idx[0]+1]]
for i in range(len(idx)- 1):
chunks.append(path[idx[i]+1:idx[i+1]])
 chunks.append(path[idx[i+1]:idx[i+1]+1])
 chunks = [chunk for chunk in chunks if len(chunk)]
scores = [chunk_scores(chunk)for chunk in chunks]
default_score = score_compound_chunk(idx[0]-1, head, chunks, tail, scores)
 best_score = default_score
for indexes_permutation in not_trivial_indexes_permutations(len(chunks)) :
score = score_compound_chunk(idx[0]-1, head, chunks, tail, scores, indexes_permutation)
 if score < best_score:
permutation = [chunks[i] for i in indexes_permutation]
best_chunk = np.concatenate([[head], np.concatenate(permutation), [tail]])
 best_score = score
if best_score < default_score:
path[idx[0]-1:idx[-1]+2] = best_chunk
path_index = np.argsort(path[:-1])
 improvement = True<compute_test_metric>
def sliding_window(img, sz=256):
total=7
x=0
img=img[0]
new_img = np.empty(( total,256,256,3),dtype=np.uint8)
 for count in range(total-1):
new_img[count,]=img[:, x:x+sz, :]
x=x+sz
new_img[-1,]=img[:, -256:, :]
return new_img
final_score = score_path(path)
 print("Final score is {:.2f}, improvement: {:.2f}".format(final_score, initial_score - final_score))<save_to_csv>
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,
batch_size=1, dim=(256, 1600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)// self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return crop_rand_pattern(X, y)
 elif self.mode == 'predict':
return sliding_window(X)
 elif self.mode=='val':
y = self.__generate_y(list_IDs_batch)
 return sliding_window_predict(X, y)
 else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
img_path = f"{self.base_path}/{im_name}"
img =cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
 img_i[i,]=img
return img_i
def __generate_y(self, ID):
img_i = np.empty(( self.batch_size,*self.dim,self.n_classes),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
img =build_masks(rles, input_shape=self.dim)
 img_i[i,]=img
return img_i
def __check_input__(self,index):
X,y=self.__getitem__(index)
 indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
ID=list_IDs_batch
X=X
y=y
for i,ID in enumerate(list_IDs_batch):
im_name=self.df['Image_id'].loc[ID]
img= cv2.imread(train_path+im_name)
 image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 img_i=mask_image(img.astype(np.uint8),masks.astype(np.uint8))
plt.figure(figsize=(10,10))
plt.imshow(img_i)
 plt.show()
col=y.shape[0]
fig,axis = plt.subplots(nrows=1,ncols=col, figsize=(20, 20))
for i in range(col):
try:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis[i].imshow(img_i)
 except IndexError:
print("IndexError : so skip")
 img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis.imshow(img_i,cmap='gray')
 break
except TypeError:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
print("i")
 axis.imshow(img_i,cmap='gray')
 print("TypeError(axis[i] is fails): so skip")
 break
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False)
 make_submission(score_path(path), path )<import_modules>
model_seg_2 = load_model('/kaggle/input/seg-model-256-256/seg_model.h5')
 model_seg_2.load_weights('/kaggle/input/seg-weights-898/bce_dice_loss_bce_from_8963_exp_4.h5' )
import numpy as np
import pandas as pd
import numba
from sympy import isprime, primerange
from math import sqrt
from sklearn.neighbors import KDTree
from tqdm import tqdm
from itertools import combinations, permutations
from functools import lru_cache<load_from_csv>
test_df = []
for i in range(0, test_imgs.shape[0], 300):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 300)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model_seg.predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 for j, b in tqdm(enumerate(batch_idx)) :
filename = test_imgs['Image_id'].iloc[b]
image_df = filtered_sub_df[filtered_sub_df['Image_id'] == filename].copy()
pred_masks=batch_pred_masks[j,].round().astype(int)
 value = np.sum(pred_masks,axis=(1,0))
c1=np.array(value*image_df['EncodedPixels'])
 pred_rles = np.array(build_rles(pred_masks))
pred_rles[c1==0]=''
image_df['EncodedPixels'] = pred_rles
test_df.append(image_df)
 gc.collect()
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<compute_test_metric>
test_df = pd.concat(test_df)
 print(test_df.shape)
 test_df.head()
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path)
 @numba.jit
def chunk_scores(chunk):
scores = np.zeros(10)
 pure_distance = 0
for i in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[i], chunk[i+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if is_not_prime[id_from]:
scores[9-i%10] += distance
scores *= 0.1
scores += pure_distance
return scores
def score_compound_chunk(offset, head, chunks, tail, scores, indexes_permutation=None):
if indexes_permutation is None:
indexes_permutation = range(len(chunks))
score = 0.0
last_city_id = head
for index in indexes_permutation:
chunk, chunk_scores = chunks[index], scores[index]
score += cities_distance(offset % 10, last_city_id, chunk[0])
 score += chunk_scores[(offset + 1)% 10]
last_city_id = chunk[-1]
offset += len(chunk)
 return score + cities_distance(offset % 10, last_city_id, tail)
<choose_model_class>
test_df[['ImageId_ClassId', 'EncodedPixels']].to_csv('submission.csv', index=False )
kdt = KDTree(XY )<load_from_csv>
a=np.array([0,2])
 a[np.where(a>0.7)]=10
a