Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<string_transform>
!pip install --no-deps.. /input/segmentations-models-pytorch-7fa1020/pretrainedmodels-0.7.4-py3-none-any.whl
sys.path.append('.. /input/smp-latest/segmentation_models.pytorch-master/')
 sys.path.append('.. /input/efficientnet-pytorch/efficientnet_pytorch-0.4.0/' )
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if fold==(FNUMBER-1):
idxTT = idxT; idxVV = idxV
print('
print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)
<define_variables>
VERBOSE = 0
DISPLAY_PLOT = True
oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold in range(EXPERIMENTS):
idxT = idxTT
idxV = idxVV
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS,
DROP_FREQ[fold],DROP_CT[fold],DROP_SIZE[fold]))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
for k in range(M1[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 print('
for k in range(M2[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT+15])
 print('
for k in range(M3[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1+30])
 print('
for k in range(M4[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+30])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold],
droprate=DROP_FREQ[fold], dropct=DROP_CT[fold], dropsize=DROP_SIZE[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*2,
droprate=DROP_FREQ[fold], dropct=DROP_CT[fold], dropsize=DROP_SIZE[fold])
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/2/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
if INFER_TEST:
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*2,
droprate=DROP_FREQ[fold], dropct=DROP_CT[fold], dropsize=DROP_SIZE[fold])
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/2/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('EXPERIMENT %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i, M1=%i, M2=%i, M3=%i, M4=%i
\
batch_size %i, dropout_freq=%.2f count=%i size=%.3f'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold],M1[fold],M2[fold],M3[fold],
M4[fold],BATCH_SIZES[fold]*REPLICAS,DROP_FREQ[fold],DROP_CT[fold],DROP_SIZE[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
del model; z = gc.collect()<save_to_csv>
df=pd.read_csv('.. /input/steel-classifier-csv/train_classes.csv')
 df.Label =(df.Label!='0' ).astype('int')
 sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_df= pd.read_csv(sample_submission_path)
 def changename(x):
x=x[:-2]
return x
test_df.ImageId_ClassId=test_df.ImageId_ClassId.apply(changename)
 test_df=test_df.drop_duplicates(['ImageId_ClassId'] )
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<save_to_csv>
test_df.EncodedPixels.value_counts()
if INFER_TEST:
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()
plt.hist(submission.target,bins=100)
 plt.show()
else:
print('We did not predict test set' )<import_modules>
import pdb
import os
import cv2
import pandas as pd
import numpy as np
from tqdm import tqdm_notebook as tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose, HorizontalFlip)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
import pickle as p
import math
import random<load_pretrained>
device = torch.device("cuda" )
def load_tour(filename):
tour = []
with open(filename)as stream:
for i, line in enumerate(stream.readlines()):
if i == 0: continue
city = int(line)
 tour.append(city)
 assert(tour[0] == 0)and(tour[-1] == 0)
 return tour
def load_pickle(filename):
with open(filename, "rb")as stream:
return p.load(stream)
 popmusic = load_pickle('.. /input/travelingsantainputdata/popmusic.pickle')
 tour = load_tour('.. /input/travelingsantainputdata/tour-1516254.csv')
 print('LKH output tour & neighbor list successfully imported.' )<randomize_order>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
def shuffle(l):
l = list(l)
 random.shuffle(l)
 return iter(l)
 class World:
def __init__(self, filename):
cities_x, cities_y, n = World._read_cities(filename)
 self.cities_x = tuple(cities_x)
 self.cities_y = tuple(cities_y)
 self.n = n
primes = World._prime_sieve(self.n)
 self.non_primes = tuple(1 - p for p in primes)
 @staticmethod
def _read_cities(filename):
cities_x, cities_y = [], []
with open(filename)as s:
for i, line in enumerate(s.readlines()):
if i == 0: continue
_, x, y = line.strip().split(',')
 cities_x.append(float(x))
cities_y.append(float(y))
assert len(cities_x)== i and len(cities_y)== i
return cities_x, cities_y, i
@staticmethod
def _prime_sieve(n):
assert n >= 2
primes = [False, False] + [True] *(n - 2)
 for i in range(2, n):
if primes[i]:
for j in range(2 * i, n, i):
primes[j] = False
return primes
def score(self, tour, percent=0.1):
distances = []
for i in range(1, len(tour)) :
x = self.cities_x[tour[i]] - self.cities_x[tour[i-1]]
y = self.cities_y[tour[i]] - self.cities_y[tour[i-1]]
distances.append(math.sqrt(( x * x)+(y * y)))
 penalties = []
for i in range(9, len(distances), 10):
penalties.append(self.non_primes[tour[i]] * distances[i])
 return sum(distances)+ percent * sum(penalties)
 class Tour:
@staticmethod
def load_csv(filename):
tour = []
with open(filename)as stream:
for i, line in enumerate(stream.readlines()):
if i == 0: continue
city = int(line)
 tour.append(city)
 assert(tour[0] == 0)and(tour[-1] == 0)
 return tour
@staticmethod
def save_csv(tour, filename):
with open(filename, "w")as s:
s.write("Path
")
 for city in tour:
s.write("%d
" % city)
 class Builder:
def noop(self):
self.add_forward(0, -1)
 return self
def forward_2opt(self, a, b):
self.add_forward(0, a)
 self.add_backward(a + 1, b)
 self.add_forward(b + 1, -1)
 return self
def forward_3opt(self, a, b, c):
self.add_forward(0, a)
 self.add_forward(b + 1, c)
 self.add_forward(a + 1, b)
 self.add_forward(c + 1, -1)
 return self
def forward_2x_bridge(self, a, b, c, d):
self.add_forward(0, a)
 self.add_forward(c + 1, d)
 self.add_forward(b + 1, c)
 self.add_forward(a + 1, b)
 self.add_forward(d + 1, -1)
 return self
class Candidates:
@staticmethod
def tour_inverse(tour):
tour_inverse = [0] * len(tour)
 for i, j in enumerate(tour):
tour_inverse[j] = i
return tour_inverse
@staticmethod
def pairs_2opt(tour, neighbors):
tour_inverse = Candidates.tour_inverse(tour)
 def n(x):
for y in neighbors[tour[x]]: yield tour_inverse[y]
for a0 in shuffle(range(len(tour)- 1)) :
a1 = a0 + 1
for b0 in n(a0):
b1 = b0 + 1
if a0 < b0 and b1 in n(a1):
yield a0, b0
@staticmethod
def triples_3opt(tour, neighbors):
tour_inverse = Candidates.tour_inverse(tour)
 def n(x):
for y in neighbors[tour[x]]: yield tour_inverse[y]
for a0 in shuffle(range(len(tour))):
a1 = a0 + 1
for b1 in n(a0):
b0 = b1 - 1
if a0 < b0:
for c1 in n(b0):
c0 = c1 - 1
if b0 < c0 and a1 in n(c0):
yield a0, b0, c0
@staticmethod
def quads_2x_bridge(tour, neighbors):
tour_inverse = Candidates.tour_inverse(tour)
 def n(x):
for y in neighbors[tour[x]]: yield tour_inverse[y]
pairs = []
for a0 in range(len(tour)- 1):
a1 = a0 + 1
for b1 in n(a0):
b0 = b1 - 1
if a0 < b0 and b0 in n(a1):
pairs.append(( a0, b0))
for i in shuffle(range(len(pairs))):
a, c = pairs[i]
quads = []
for j in range(i + 1, len(pairs)) :
b, d = pairs[j]
if not a < b: continue
if not b < c: break
if c < d: quads.append(( a, b, c, d))
for quad in shuffle(quads):
yield quad
class Scorer:
def __init__(self, tour, world, percent=0.1):
self.tour = tour
self.world = world
self.percent = percent
self.forward_scores = self._preprocess(tour)
 self.backward_scores = self._preprocess(tour[::-1])
 def _preprocess(self, tour):
distances = []
for i in range(1, len(tour)) :
x = self.world.cities_x[tour[i]] - self.world.cities_x[tour[i-1]]
y = self.world.cities_y[tour[i]] - self.world.cities_y[tour[i-1]]
distances.append(math.sqrt(( x * x)+(y * y)))
 scores = []
for j in range(10):
penalties = [0] * len(distances)
 for k in range(j, len(distances), 10):
penalties[k] = self.percent * self.world.non_primes[tour[k]] * distances[k]
cumsum = []
cumsum.append(distances[0] + penalties[0])
 for k in range(1, len(distances)) :
cumsum.append(cumsum[-1] + distances[k] + penalties[k])
 scores.append(cumsum)
 return scores
def score_forward(self, a, b, new_start):
assert b >= a
offset =(a - new_start + 9)% 10
distance = self.forward_scores[offset][b - 1]
if a > 0: distance -= self.forward_scores[offset][a - 1]
return distance
def score_edge(self, i, j, edge_index):
assert i != j
x = self.world.cities_x[j] - self.world.cities_x[i]
y = self.world.cities_y[j] - self.world.cities_y[i]
distance = math.sqrt(x * x + y * y)
 if edge_index % 10 == 9 and self.world.non_primes[i]:
distance *=(1 + self.percent)
 return distance
class ScoreBuilder(Builder):
def __init__(self, scorer):
self.scorer = scorer
self.score = 0.0
self.tour_length = 0
self.last_city = None
def add_forward(self, a, b):
if b == -1: b = len(self.scorer.tour)- 1
assert b >= a
if self.tour_length > 0:
self.score += self.scorer.score_edge(
 self.last_city,
self.scorer.tour[a],
self.tour_length - 1)
 if b > a:
self.score += self.scorer.score_forward(
 a, b, self.tour_length)
 self.tour_length += b - a + 1
self.last_city = self.scorer.tour[b]
def out(self):
return self.score
class TourBuilder(Builder):
def __init__(self, tour):
self.tour = tour
self.new_tour = []
def add_forward(self, a, b):
if b == -1: b = len(self.tour)- 1
assert b >= a
self.new_tour.extend(self.tour[a:b+1])
 def out(self):
return self.new_tour
class Neighbors:
@staticmethod
def load_pickle(filename, limit=None):
with open(filename, "rb")as stream:
neighbors = p.load(stream)
 if limit is not None:
for i in range(len(neighbors)) :
neighbors[i] = neighbors[i][:limit]
for i in range(len(neighbors)) :
for j in neighbors[i]:
if i not in neighbors[j]:
neighbors[j].append(i)
 return neighbors
class Move:
def __init__(self, name, candidates, score_forward, tour_forward):
self.name = name
self.candidates = candidates
self.score_forward = score_forward
self.tour_forward = tour_forward
move_3opt = Move(
 name="3-OPT",
candidates=lambda t: Candidates.triples_3opt(t, popmusic),
score_forward=lambda s, c: ScoreBuilder(s ).forward_3opt(*c ).out() ,
tour_forward=lambda t, c: TourBuilder(t ).forward_3opt(*c ).out() , 
)
 move_2x_bridge = Move(
 name="DOUBLE BRIDGE",
candidates=lambda t: Candidates.quads_2x_bridge(t, popmusic),
score_forward=lambda s, c: ScoreBuilder(s ).forward_2x_bridge(*c ).out() ,
tour_forward=lambda t, c: TourBuilder(t ).forward_2x_bridge(*c ).out() , 
)
 def progress_one(tour, world, move, percent=0.1, epsilon=1e-3):
scorer = Scorer(tour, world, percent)
 base_score = ScoreBuilder(scorer ).noop().out()
for candidate in move.candidates(tour):
score = move.score_forward(scorer, candidate)
 if score < base_score - epsilon:
return move.tour_forward(tour, candidate)
 def progress_many(tour, world, move, percent=0.1, epsilon=1e-3, limit=None, verbose=5):
counter = 0
while True:
counter += 1
if limit is not None and counter > limit: return tour
if verbose is not None:
if counter % verbose == 0: print(counter, world.score(tour, percent))
previous_tour = tour
tour = progress_one(previous_tour, world, move, percent, epsilon)
 if tour is None:
return previous_tour
def progress_all(tour, world, moves, percent=0.1, epsilon=1e-3, verbose=20):
counter = 0
while True:
for move in moves:
previous_tour = tour
tour = progress_many(tour, world, move, percent, epsilon, verbose=verbose)
 counter = 0 if tour != previous_tour else counter + 1
if counter == len(moves)- 1:
return tour
def kick(tour, world, move):
limit = int(random.uniform(50,100))
epsilon = - random.uniform(0.1, 0.5)
 penalty = 0.1
print("Begin kicking...epsilon =", epsilon, ', number of kicks =', limit, ', kick penalty =', penalty)
 return progress_many(tour, world, move, percent=penalty, epsilon=epsilon, limit=limit, verbose=None)
 print('Successfully importing custom library' )<find_best_params>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor()
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
VERBOSE = 5
world = World('.. /input/traveling-santa-2018-prime-paths/cities.csv')
 print('INPUT SCORE:', world.score(tour))
for penalty in [0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.1]:
for epsilon in [0]:
print('Penalty:', penalty, '.Epsilon:', epsilon)
 tour = progress_all(tour, world,(move_2x_bridge, move_3opt), percent=penalty, epsilon=epsilon, verbose=VERBOSE)
 print('SCORE AFTER OPTIMIZATION:', world.score(tour)) 
<compute_test_metric>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
nb_kicks = 10
verbose = 1
kick_move = move_2x_bridge
reoptimize_moves =(move_3opt , move_2x_bridge)
 cnt_success = 0
cnt_fail = 0
for k in range(nb_kicks):
input_score = world.score(tour)
 print('Kick', k ,'INPUT SCORE:', input_score)
 broken_tour = kick(tour, world, kick_move)
 broken_score = world.score(broken_tour)
 print("Kicked.Score increase", broken_score - input_score, '(', broken_score ,')')
 print("Now fixing...")
 fixed_tour = progress_all(broken_tour, world, reoptimize_moves)
 score = world.score(fixed_tour)
 if score < input_score:
cnt_success += 1
print('KICK SUCCESSFUL! New score:', score)
 filename = "tour-" + "%.2f" % score + ".csv"
Tour.save_csv(tour, filename)
 tour = fixed_tour
else:
print('KICK FAILED!')
 cnt_fail += 1
print('Success count:', cnt_success, 'Fail count:', cnt_fail )<load_pretrained>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
class World:
def __init__(self, filename):
cities_x, cities_y, n = World._read_cities(filename)
 self.cities_x = tuple(cities_x)
 self.cities_y = tuple(cities_y)
 self.n = n
primes = World._prime_sieve(self.n)
 self.non_primes = tuple(1 - p for p in primes)
 @staticmethod
def _read_cities(filename):
cities_x, cities_y = [], []
with open(filename)as s:
for i, line in enumerate(s.readlines()):
if i == 0: continue
_, x, y = line.strip().split(',')
 cities_x.append(float(x))
cities_y.append(float(y))
assert len(cities_x)== i and len(cities_y)== i
return cities_x, cities_y, i
@staticmethod
def _prime_sieve(n):
assert n >= 2
primes = [False, False] + [True] *(n - 2)
 for i in range(2, n):
if primes[i]:
for j in range(2 * i, n, i):
primes[j] = False
return primes
def distance(self, i, j):
x = self.cities_x[i] - self.cities_x[j]
y = self.cities_y[i] - self.cities_y[j]
return math.sqrt(( x * x)+(y * y))
def score(self, tour, percent=0.1):
distances = []
for i in range(1, len(tour)) :
distances.append(self.distance(tour[i], tour[i-1]))
penalties = []
for i in range(9, len(tour)- 1, 10):
penalties.append(self.non_primes[tour[i]] * distances[i])
 return sum(distances)+ percent * sum(penalties)
 class Tour:
@staticmethod
def load_csv(filename):
tour = []
with open(filename)as stream:
for i, line in enumerate(stream.readlines()):
if i == 0: continue
city = int(line)
 tour.append(city)
 assert(tour[0] == 0)and(tour[-1] == 0)
 return tour
@staticmethod
def save_csv(tour, filename):
with open(filename, "w")as s:
s.write("Path
")
 for city in tour:
s.write("%d
" % city)
 @staticmethod
def edge_distance(tour1, tour2):
assert len(tour1)== len(tour2)
 def edges(tour):
for i in range(len(tour)- 1):
yield tuple(sorted(( tour[i], tour[i + 1])))
 edges2 = set(edges(tour2))
distance = 0
for edge in edges(tour1):
if edge not in edges2:
distance += 1
return distance
def prime_distance(tour1, tour2):
assert len(tour1)== len(tour2)
 primes = World._prime_sieve(len(tour1))
def placed_primes(tour):
for i in range(9, len(tour)- 1, 10):
if primes[tour[i]]: yield tour[i]
primes1 = set(placed_primes(tour1))
primes2 = set(placed_primes(tour2))
return len(primes1 | primes2)- len(primes1 & primes2)
 @staticmethod
def distance(tour1, tour2):
return(
 Tour.edge_distance(tour1, tour2),
Tour.prime_distance(tour1, tour2))
class Neighbors:
@staticmethod
def load_pickle(filename):
with open(filename, "rb")as stream:
return p.load(stream)
<feature_engineering>
test_df_= pd.read_csv(sample_submission_path )
class Dict:
def __init__(self):
self.index = dict()
self.keys = list()
self.values = list()
def __contains__(self, key):
return key in self.index
def __len__(self):
return len(self.keys)
 def __setitem__(self, key, value):
assert key not in self
self.index[key] = len(self.keys)
 self.keys.append(key)
 self.values.append(value)
 def __getitem__(self, key):
return self.values[self.index[key]]
def __delitem__(self, key):
assert key in self
i = self.index[key]
k = self.keys[-1]
v = self.values[-1]
self.index[k] = i
self.keys[i] = k
self.values[i] = v
del self.index[key]
del self.keys[-1]
del self.values[-1]
def random_key(self):
return random.choice(self.keys)
 class Graph:
def __init__(self):
self.edges = Dict()
def empty(self):
return len(self.edges)== 0
def add_edge(self, v0, v1):
if v0 not in self.edges:
self.edges[v0] = list()
self.edges[v0].append(v1)
 if v1 not in self.edges:
self.edges[v1] = list()
self.edges[v1].append(v0)
 def remove_edge(self, v0, v1):
self.edges[v0].remove(v1)
 if len(self.edges[v0])== 0:
del self.edges[v0]
self.edges[v1].remove(v0)
 if len(self.edges[v1])== 0:
del self.edges[v1]
def random_vertex(self):
assert not self.empty()
return self.edges.random_key()
def random_adjacent(self, v):
assert v in self.edges
return random.choice(self.edges[v])
 @staticmethod
def from_subtours(subtours):
graph = Graph()
for subtour in subtours:
assert subtour[0] == subtour[-1]
for i in range(len(subtour)- 1):
v0, v1 = subtour[i], subtour[i + 1]
graph.add_edge(v0, v1)
 return graph
def to_subtours(self, v=0):
while not self.empty() :
if v not in self.edges:
v = self.random_vertex()
subtour = [v]
while True:
v0 = subtour[-1]
if v0 not in self.edges: break
v1 = self.random_adjacent(v0)
 self.remove_edge(v0, v1)
 subtour.append(v1)
 assert subtour[0] == subtour[-1]
yield subtour
class AB_Cycles:
@staticmethod
def _find_cycle(path):
assert len(path)> 0
last_vertex = path[-1]
for index in range(len(path)- 3, -1, -2):
if path[index] == last_vertex:
return index
@staticmethod
def _remove_cycle(path):
index = AB_Cycles._find_cycle(path)
 if index is not None:
path, cycle = path[:(index + 1)], path[index:]
if len(path)& 1 == 0: cycle = cycle[1:] + [cycle[1]]
if len(path)== 1: path = []
else: cycle = None
return path, cycle
@staticmethod
def ab_cycles(tour_a, tour_b):
assert len(tour_a)== len(tour_b)
 path = list()
graphs =(
 Graph.from_subtours([tour_a]),
Graph.from_subtours([tour_b]))
while True:
if len(path)== 0:
if graphs[0].empty() : break
vertex = graphs[0].random_vertex()
path.append(vertex)
 ab = 1 -(len(path)& 1)
 v0 = path[-1]
v1 = graphs[ab].random_adjacent(v0)
 graphs[ab].remove_edge(v0, v1)
 path.append(v1)
 path, cycle = AB_Cycles._remove_cycle(path)
 if cycle is not None: yield cycle
assert graphs[0].empty()
assert graphs[1].empty()
assert len(path)== 0
class EAX:
@staticmethod
def _eset_rand(ab_cycles):
for ab_cycle in ab_cycles:
if len(ab_cycle)> 3 and random.uniform(0, 1)> 0.5:
yield ab_cycle
@staticmethod
def _intermediate_solution(tour_a, eset):
graph = Graph.from_subtours([tour_a])
 for ab_cycle in eset:
ab = 0
for i in range(len(ab_cycle)- 1):
v0, v1 = ab_cycle[i], ab_cycle[i + 1]
if ab == 0: graph.remove_edge(v0, v1)
 else: graph.add_edge(v0, v1)
 ab = 1 - ab
return graph
@staticmethod
def _shortest_subtour(subtours):
lengths = [len(subtour)for subtour in subtours]
i = lengths.index(min(lengths))
shortest = subtours[i]
others = subtours[:i] + subtours[(i + 1):]
return shortest, others
@staticmethod
def _quads(subtours, neighbors):
shortest, others = EAX._shortest_subtour(subtours)
 x = list()
for i in range(len(shortest)- 1):
v0, v1 = shortest[i], shortest[i + 1]
x.append(( v0, v1))
y = set()
for subtour in others:
for i in range(len(subtour)- 1):
v0, v1 = subtour[i], subtour[i + 1]
y.add(( v0, v1))
y.add(( v1, v0))
for v0, v1 in x:
for v2 in neighbors[v0]:
for v3 in neighbors[v1]:
if(v2, v3)in y:
yield v0, v1, v2, v3
@staticmethod
def _min_quad(subtours, world, neighbors):
min_distance = None
min_quad = None
for v0, v1, v2, v3 in EAX._quads(subtours, neighbors):
removed = world.distance(v0, v1)+ world.distance(v2, v3)
 for(v2, v3)in(( v2, v3),(v3, v2)) :
added = world.distance(v0, v2)+ world.distance(v1, v3)
 distance = added - removed
if min_distance is None or distance < min_distance:
min_distance = distance
min_quad = v0, v1, v2, v3
return min_quad
def _offspring_solution(graph, world, neighbors):
while True:
subtours = tuple(graph.to_subtours())
 if len(subtours)== 1: return subtours[0]
print("subtours", len(subtours))
v0, v1, v2, v3 = EAX._min_quad(subtours, world, neighbors)
 graph = Graph.from_subtours(subtours)
 graph.remove_edge(v0, v1)
 graph.remove_edge(v2, v3)
 graph.add_edge(v0, v2)
 graph.add_edge(v1, v3)
 @staticmethod
def eax(tour_a, tour_b, world, neighbors):
ab_cycles = AB_Cycles.ab_cycles(tour_a, tour_b)
 eset = EAX._eset_rand(ab_cycles)
 intermediate = EAX._intermediate_solution(tour_a, eset)
 return EAX._offspring_solution(intermediate, world, neighbors)
<init_hyperparams>
fold_csv = pd.read_csv(sample_submission_path)
 num_workers = 4
batch_size = 4
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, test_df_, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)
%%sh
cat > main.cpp <<'EOF'
using namespace std;
constexpr int maxShift = 3; //5
constexpr int cntNeighbors = 12;
constexpr int cntCities = 197769;
class GlobalIndex {
public:
array<double, 10> penalty_coefficients;
vector<vector<int>> neighbors;
bitset<cntCities> primes;
array<double, cntCities> x, y;
GlobalIndex()
: neighbors(cntCities+1, vector<int>(100)) {
penalty_coefficients.fill(1.0);
penalty_coefficients[0] = 1.1;
primes.set() ;
primes[0] = primes[1] = false;
for(int p = 2; p * p <= cntCities; ++p)if(primes[p])
 for(int i = p*2; i < cntCities; i += p)
 primes[i] = false;
// Read coordinates
ifstream file(".. /input/traveling-santa-2018-prime-paths/cities.csv");
assert(file && "cities.csv");
file.ignore(numeric_limits<streamsize>::max() , '
');
char delimiter;
double X, Y;
for(int i; file >> i >> delimiter >> X >> delimiter >> Y;){
x[i] = X;
y[i] = Y;
}
// Read neighbors for not depends from libboost
ifstream file2(".. /input/list-100-neighbors-each-cities/neighbors100.bin", ios::binary);
assert(file2 && "neighbors100.bin");
for(auto& arr : neighbors)
 file2.read(reinterpret_cast<char*>(arr.data()), arr.size() *sizeof(arr[0])) ;
}
inline double dist(int l, int r)const {
const double dx = x[l] - x[r];
const double dy = y[l] - y[r];
return sqrt(dx*dx + dy*dy);
}
inline double dist(int l, int r, int i)const {
if(primes[l])return dist(l, r);
return penalty_coefficients[(i+1)%10] * dist(l, r);
}
};
const GlobalIndex gi;
class Index {
public:
explicit Index(const vector<int>& ids){
positions.resize(ids.size());
for(auto& diff_distance : diff_distances)
 diff_distance.resize(ids.size());
for(auto& diff_distance : rdiff_distances)
 diff_distance.resize(ids.size());
pure_dist.resize(ids.size() -1);
Update(ids);
}
void Update(const vector<int>& ids){
for(int i = 0; i < ids.size() ; ++i)
 positions[ids[i]] = i;
for(int i = 0; i < pure_dist.size() ; ++i)
 pure_dist[i] = gi.dist(ids[i], ids[i+1]);
for(int j = 0; j < diff_distances.size() ; ++j)
 for(int i = 1; i < diff_distances[0].size() ; ++i){
double value = pure_dist[i-1];
if(!gi.primes[ids[i-1]])
 value *= gi.penalty_coefficients[(i+j)%10];
diff_distances[j][i] = diff_distances[j][i-1] + value;
}
for(int j = 0; j < rdiff_distances.size() ; ++j)
 for(int i = 1, sz = rdiff_distances[0].size() ; i < sz; ++i){
double value = pure_dist[i-1];
if(!gi.primes[ids[i]])
 value *= gi.penalty_coefficients[(sz-i+j+1)%10];
rdiff_distances[j][i] = rdiff_distances[j][i-1] + value;
}
}
array<vector<double>, 10> diff_distances;
array<vector<double>, 10> rdiff_distances;
vector<int> positions;
vector<double> pure_dist;
inline double getScore(const vector<int>& ids, int beg, int end)const {
return diff_distances[0][end] - diff_distances[0][beg];
}
inline double getScoreReverse(const vector<int>& ids, int beg, int end, int shift = 0)const {
int i =(beg + end + shift)%10;
return rdiff_distances[i][end] - rdiff_distances[i][beg];
}
inline double getScore(const vector<int>& ids, int beg, int end, int shift)const {
if(beg <= end){
int i = shift ?(cntCities + shift + 1)%10 : 0;
return diff_distances[i][end] - diff_distances[i][beg];
}
return getScoreReverse(ids, end, beg, shift);
}
double getScore() const {
return diff_distances[0].back() ;
}
};
auto loadIds(const string& name){
vector<int> result;
ifstream file(name);
assert(file);
file.ignore(numeric_limits<streamsize>::max() , '
');
int id;
while(file >> id)
 result.emplace_back(id);
return result;
}
void saveIds(const vector<int>& ids, const string& name){
ofstream file(name);
file << "Path";
for(const auto& id : ids)
 file << endl << id;
}
auto compareReverse = [](const Index& index, const vector<int>& ids, int beg, int end){
if(end - beg < 3)
 return numeric_limits<double>::lowest() ;
double sum = index.getScore(ids, beg, end);
sum -= index.getScoreReverse(ids, beg + 1, end - 1);
sum -= gi.dist(ids[beg], ids[end - 1], beg);
sum -= gi.dist(ids[beg + 1], ids[end], end - 1);
return sum;
};
auto compareShiftRightN = [](const int n){
return [n](const Index& index, const vector<int>& ids, int beg, int end){
if(end >= ids.size() - 1 || end - beg < n + 1)
 return numeric_limits<double>::lowest() ;
double sum = index.getScore(ids, beg, end + 1);
sum -= index.getScore(ids, beg + 1, end - n, n);
for(auto i = 0; i < n - 1; ++i)
 sum -= gi.dist(ids[end - i], ids[end - i - 1], beg + i + 1);
sum -= gi.dist(ids[beg], ids[end], beg);
sum -= gi.dist(ids[end - n + 1], ids[beg + 1], beg+n);
sum -= gi.dist(ids[end - n], ids[end + 1], end);
return sum;
};
};
auto compareShiftLeftN = [](const int n){
return [n](const Index& index, const vector<int>& ids, int beg, int end){
if(beg == 0 || end - beg < n + 1)
 return numeric_limits<double>::lowest() ;
double sum = index.getScore(ids, beg - 1, end);
sum -= index.getScore(ids, beg+n, end - 1, -n);
for(auto i = 0; i < n - 1; ++i)
 sum -= gi.dist(ids[beg + 1 + i], ids[beg + i], end - 2 - i);
sum -= gi.dist(ids[beg - 1], ids[beg + n] , beg - 1);
sum -= gi.dist(ids[end - 1], ids[beg + n - 1], end - n - 1);
sum -= gi.dist(ids[beg] , ids[end] , end - 1);
return sum;
};
};
struct Result {
Result(int l, int r): beg(min(l ,r)) , end(max(l, r)) {}
double dist;
int beg, end;
friend bool operator<(const Result& l, const Result& r){return l.dist < r.dist;}
};
auto bestPairNeighbors = [](const Index& index, const vector<int>& ids, auto comp){
Result best(0, cntCities);
best.dist = comp(index, ids, best.beg, best.end);
for(int it = 0; it < ids.size() ; ++it)
 for(int i = 0; i < cntNeighbors; ++i){
Result tmp(it, index.positions[gi.neighbors[ids[it]][i]]);
tmp.dist = comp(index, ids, tmp.beg, tmp.end);
best = max(tmp, best);
}
return best;
};
auto shiftLeftN = [](const int n){
return [n](vector<int>& ids, int beg, int end){
rotate(begin(ids)+ beg, begin(ids)+ beg + n, begin(ids)+end);
reverse(begin(ids)+ end - n, begin(ids)+ end);
};
};
auto shiftRightN = [](const int n){
return [n](vector<int>& ids, int beg, int end){
auto it = rbegin(ids)+ ids.size() - 1;
rotate(it - end, it - end + n, it - beg);
reverse(begin(ids)+ beg + 1, begin(ids)+ beg + n + 1);
};
};
void reverseMover(vector<int>& ids, int beg, int end){
reverse(begin(ids)+ beg + 1, begin(ids)+ end);
}
template<class Func>
void fastMover(vector<int>& ids, Func func){
cout << fixed << setprecision(4);
vector<function<double(const Index&, const vector<int>&, int, int)>> comparators{compareReverse};
vector<function<void(vector<int>&, int, int)>> movers{reverseMover};
for(int i = 1; i <= maxShift; ++i){
comparators.emplace_back(compareShiftLeftN(i)) ;
comparators.emplace_back(compareShiftRightN(i)) ;
movers.emplace_back(shiftLeftN(i)) ;
movers.emplace_back(shiftRightN(i)) ;
}
Index index(ids);
while(true){
vector<Result> results;
vector<future<Result>> futures;
for(auto& comp : comparators)
 futures.emplace_back(async(func, index, ids, comp)) ;
for(auto& item : futures)
 results.emplace_back(item.get());
auto best = max_element(begin(results), end(results)) - begin(results);
if(results[best].dist < 0.0001)
 return;
movers[best](ids, results[best].beg, results[best].end);
index.Update(ids);
cout << "Score: " << index.getScore() << setw(10)<< -results[best].dist << endl;
}
}
int main(int argc, char** argv){
if(argc < 2){
cerr << "Usage: " << argv[0] << " <input.tour> <output.tour>" << endl;
return 1;
}
auto ids = loadIds(argv[1]);
fastMover(ids, bestPairNeighbors);
saveIds(ids, argv[2]);
return 0;
}
EOF<set_options>
GlobalParams = namedtuple('GlobalParams', ['batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'num_classes',
'width_coefficient', 'depth_coefficient', 'depth_divisor', 'min_depth',
'drop_connect_rate'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs = namedtuple('BlockArgs', ['kernel_size', 'num_repeat', 'input_filters', 'output_filters', 'expand_ratio',
'id_skip', 'strides', 'se_ratio'])
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 IMAGENET_WEIGHTS = {
'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',
'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',
'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth'
}
def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
if not multiplier:
return filters
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def get_efficientnet_params(model_name, override_params=None):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
}
if model_name not in params_dict.keys() :
raise KeyError('There is no model named {}.'.format(model_name))
width_coefficient, depth_coefficient, _, dropout_rate = params_dict[model_name]
blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=0.2,
num_classes=1000,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None)
 if override_params:
global_params = global_params._replace(**override_params)
 decoder = BlockDecoder()
return decoder.decode(blocks_args), global_params
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
if 's' not in options or len(options['s'])!= 2:
raise ValueError('Strides options should be a pair of integers.')
 return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
strides=[int(options['s'][0]), int(options['s'][1])] 
)
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 def decode(self, string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(self._decode_block_string(block_string))
return blocks_args
def encode(self, blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(self._encode_block_string(block))
return block_strings
class EfficientNet(nn.Module):
def __init__(self, block_args_list, global_params):
super().__init__()
self.block_args_list = block_args_list
self.global_params = global_params
batch_norm_momentum = 1 - self.global_params.batch_norm_momentum
batch_norm_epsilon = self.global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self.global_params)
 self._conv_stem = Conv2dSamePadding(in_channels,
out_channels,
kernel_size=3,
stride=2,
bias=False,
name='stem_conv')
 self._bn0 = BatchNorm2d(num_features=out_channels,
momentum=batch_norm_momentum,
eps=batch_norm_epsilon,
name='stem_batch_norm')
 self._swish = Swish(name='swish')
 idx = 0
self._blocks = nn.ModuleList([])
 for block_args in self.block_args_list:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self.global_params),
output_filters=round_filters(block_args.output_filters, self.global_params),
num_repeat=round_repeats(block_args.num_repeat, self.global_params)
)
 self._blocks.append(MBConvBlock(block_args, self.global_params, idx=idx))
idx += 1
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, strides=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self.global_params, idx=idx))
idx += 1
in_channels = block_args.output_filters
out_channels = round_filters(1280, self.global_params)
 self._conv_head = Conv2dSamePadding(in_channels,
out_channels,
kernel_size=1,
bias=False,
name='head_conv')
 self._bn1 = BatchNorm2d(num_features=out_channels,
momentum=batch_norm_momentum,
eps=batch_norm_epsilon,
name='head_batch_norm')
 self.dropout_rate = self.global_params.dropout_rate
self._fc = nn.Linear(out_channels, self.global_params.num_classes)
 def forward(self, x):
x = self._conv_stem(x)
 x = self._bn0(x)
 x = self._swish(x)
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self.global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= idx / len(self._blocks)
 x = block(x, drop_connect_rate)
 x = self._conv_head(x)
 x = self._bn1(x)
 x = self._swish(x)
 x = F.adaptive_avg_pool2d(x, 1 ).squeeze(-1 ).squeeze(-1)
 if self.dropout_rate > 0:
x = F.dropout(x, p=self.dropout_rate, training=self.training)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, *, n_classes=1000, pretrained=False):
return _get_model_by_name(model_name, classes=n_classes, pretrained=pretrained)
 @classmethod
def encoder(cls, model_name, *, pretrained=False):
model = cls.from_name(model_name, pretrained=pretrained)
 class Encoder(nn.Module):
def __init__(self):
super().__init__()
self.name = model_name
self.global_params = model.global_params
self.stem_conv = model._conv_stem
self.stem_batch_norm = model._bn0
self.stem_swish = Swish(name='stem_swish')
 self.blocks = model._blocks
self.head_conv = model._conv_head
self.head_batch_norm = model._bn1
self.head_swish = Swish(name='head_swish')
 def forward(self, x):
x = self.stem_conv(x)
 x = self.stem_batch_norm(x)
 x = self.stem_swish(x)
 for idx, block in enumerate(self.blocks):
drop_connect_rate = self.global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= idx / len(self.blocks)
 x = block(x, drop_connect_rate)
 x = self.head_conv(x)
 x = self.head_batch_norm(x)
 x = self.head_swish(x)
 return x
return Encoder()
@classmethod
def custom_head(cls, model_name, *, n_classes=1000, pretrained=False):
if n_classes == 1000:
return cls.from_name(model_name, n_classes=n_classes, pretrained=pretrained)
 else:
class CustomHead(nn.Module):
def __init__(self, out_channels):
super().__init__()
self.encoder = cls.encoder(model_name, pretrained=pretrained)
 self.custom_head = custom_head(self.n_channels * 2, out_channels)
 @property
def n_channels(self):
n_channels_dict = {'efficientnet-b0': 1280, 'efficientnet-b1': 1280, 'efficientnet-b2': 1408,
'efficientnet-b3': 1536, 'efficientnet-b4': 1792, 'efficientnet-b5': 2048,
'efficientnet-b6': 2304, 'efficientnet-b7': 2560}
return n_channels_dict[self.encoder.name]
def forward(self, x):
x = self.encoder(x)
 mp = nn.AdaptiveMaxPool2d(output_size=(1, 1))(x)
 ap = nn.AdaptiveAvgPool2d(output_size=(1, 1))(x)
 x = torch.cat([mp, ap], dim=1)
 x = x.view(x.size(0), -1)
 x = self.custom_head(x)
 return x
return CustomHead(n_classes)
 def _get_model_by_name(model_name, classes=1000, pretrained=False):
block_args_list, global_params = get_efficientnet_params(model_name, override_params={'num_classes': classes})
 model = EfficientNet(block_args_list, global_params)
 try:
if pretrained:
pretrained_state_dict = load_state_dict_from_url(IMAGENET_WEIGHTS[model_name])
 if classes != 1000:
random_state_dict = model.state_dict()
pretrained_state_dict['_fc.weight'] = random_state_dict['_fc.weight']
pretrained_state_dict['_fc.bias'] = random_state_dict['_fc.bias']
model.load_state_dict(pretrained_state_dict)
 except KeyError as e:
print(f"NOTE: Currently model {e} doesn't have pretrained weights, therefore a model with randomly initialized"
" weights is returned.")
 return model
class Swish(nn.Module):
def __init__(self, name=None):
super().__init__()
self.name = name
def forward(self, x):
return x * torch.sigmoid(x)
 class Conv2dSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True, name=None):
super().__init__(in_channels, out_channels, kernel_size, stride, padding=0, dilation=dilation, groups=groups,
bias=bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
self.name = name
def forward(self, x):
input_h, input_w = x.size() [2:]
kernel_h, kernel_w = self.weight.size() [2:]
stride_h, stride_w = self.stride
output_h, output_w = math.ceil(input_h / stride_h), math.ceil(input_w / stride_w)
 pad_h = max(( output_h - 1)* self.stride[0] +(kernel_h - 1)* self.dilation[0] + 1 - input_h, 0)
 pad_w = max(( output_w - 1)* self.stride[1] +(kernel_w - 1)* self.dilation[1] + 1 - input_w, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class BatchNorm2d(nn.BatchNorm2d):
def __init__(self, num_features, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True, name=None):
super().__init__(num_features, eps=eps, momentum=momentum, affine=affine,
track_running_stats=track_running_stats)
 self.name = name
def drop_connect(inputs, drop_connect_rate, training):
if not training:
return inputs
batch_size = inputs.shape[0]
keep_prob = 1.0 - drop_connect_rate
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params, idx):
super().__init__()
block_name = 'blocks_' + str(idx)+ '_'
self.block_args = block_args
self.batch_norm_momentum = 1 - global_params.batch_norm_momentum
self.batch_norm_epsilon = global_params.batch_norm_epsilon
self.has_se =(self.block_args.se_ratio is not None)and(0 < self.block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
self.swish = Swish(block_name + '_swish')
 in_channels = self.block_args.input_filters
out_channels = self.block_args.input_filters * self.block_args.expand_ratio
if self.block_args.expand_ratio != 1:
self._expand_conv = Conv2dSamePadding(in_channels=in_channels,
out_channels=out_channels,
kernel_size=1,
bias=False,
name=block_name + 'expansion_conv')
 self._bn0 = BatchNorm2d(num_features=out_channels,
momentum=self.batch_norm_momentum,
eps=self.batch_norm_epsilon,
name=block_name + 'expansion_batch_norm')
 kernel_size = self.block_args.kernel_size
strides = self.block_args.strides
self._depthwise_conv = Conv2dSamePadding(in_channels=out_channels,
out_channels=out_channels,
groups=out_channels,
kernel_size=kernel_size,
stride=strides,
bias=False,
name=block_name + 'depthwise_conv')
 self._bn1 = BatchNorm2d(num_features=out_channels,
momentum=self.batch_norm_momentum,
eps=self.batch_norm_epsilon,
name=block_name + 'depthwise_batch_norm')
 if self.has_se:
num_squeezed_channels = max(1, int(self.block_args.input_filters * self.block_args.se_ratio))
self._se_reduce = Conv2dSamePadding(in_channels=out_channels,
out_channels=num_squeezed_channels,
kernel_size=1,
name=block_name + 'se_reduce')
 self._se_expand = Conv2dSamePadding(in_channels=num_squeezed_channels,
out_channels=out_channels,
kernel_size=1,
name=block_name + 'se_expand')
 final_output_channels = self.block_args.output_filters
self._project_conv = Conv2dSamePadding(in_channels=out_channels,
out_channels=final_output_channels,
kernel_size=1,
bias=False,
name=block_name + 'output_conv')
 self._bn2 = BatchNorm2d(num_features=final_output_channels,
momentum=self.batch_norm_momentum,
eps=self.batch_norm_epsilon,
name=block_name + 'output_batch_norm')
 def forward(self, x, drop_connect_rate=None):
identity = x
if self.block_args.expand_ratio != 1:
x = self._expand_conv(x)
 x = self._bn0(x)
 x = self.swish(x)
 x = self._depthwise_conv(x)
 x = self._bn1(x)
 x = self.swish(x)
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self.swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self.block_args.input_filters, self.block_args.output_filters
if self.id_skip and self.block_args.strides == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, drop_connect_rate=drop_connect_rate, training=self.training)
 x = x + identity
return x
def double_conv(in_channels, out_channels):
return nn.Sequential(
 nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True),
nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)
)
 def up_conv(in_channels, out_channels):
return nn.ConvTranspose2d(
 in_channels, out_channels, kernel_size=2, stride=2 
)
 def custom_head(in_channels, out_channels):
return nn.Sequential(
 nn.Dropout() ,
nn.Linear(in_channels, 512),
nn.ReLU(inplace=True),
nn.Dropout() ,
nn.Linear(512, out_channels)
)
 __all__ = ['EfficientUnet', 'get_efficientunet_b0', 'get_efficientunet_b1', 'get_efficientunet_b2',
'get_efficientunet_b3', 'get_efficientunet_b4', 'get_efficientunet_b5', 'get_efficientunet_b6',
'get_efficientunet_b7']
def get_blocks_to_be_concat(model, x):
shapes = set()
blocks = OrderedDict()
hooks = []
count = 0
def register_hook(module):
def hook(module, input, output):
try:
nonlocal count
if module.name == f'blocks_{count}_output_batch_norm':
count += 1
shape = output.size() [-2:]
if shape not in shapes:
shapes.add(shape)
 blocks[module.name] = output
elif module.name == 'head_swish':
blocks.popitem()
blocks[module.name] = output
except AttributeError:
pass
if(
 not isinstance(module, nn.Sequential)
 and not isinstance(module, nn.ModuleList)
 and not(module == model)
):
hooks.append(module.register_forward_hook(hook))
model.apply(register_hook)
 model(x)
 for h in hooks:
h.remove()
return blocks
class Attention_block(nn.Module):
def __init__(self,F_g,F_l,F_int):
super(Attention_block,self ).__init__()
self.W_g = nn.Sequential(
 nn.Conv2d(F_g, F_int, kernel_size=1,stride=1,padding=0,bias=True),
nn.BatchNorm2d(F_int)
)
 self.W_xa = nn.Sequential(
 nn.Conv2d(F_l, F_int, kernel_size=1,stride=1,padding=0,bias=True),
nn.BatchNorm2d(F_int)
)
 self.psi = nn.Sequential(
 nn.Conv2d(F_int, 1, kernel_size=1,stride=1,padding=0,bias=True),
nn.BatchNorm2d(1),
nn.Sigmoid() 
)
 self.relu = nn.ReLU(inplace=True)
 def forward(self,g,xa):
g1 = self.W_g(g)
 x1 = self.W_xa(xa)
 psi = self.relu(g1+x1)
 psi = self.psi(psi)
 return xa*psi
class EfficientUnet(nn.Module):
def __init__(self, encoder, out_channels=2, concat_input=True):
super().__init__()
self.encoder = encoder
self.concat_input = concat_input
self.up_conv1 = up_conv(self.n_channels, 512)
 self.Att1 = Attention_block(F_g=512,F_l=592-512,F_int=56)
 self.double_conv1 = double_conv(self.size[0], 512)
 self.up_conv2 = up_conv(512, 256)
 self.Att2 = Attention_block(F_g=256,F_l=296-256,F_int=28)
 self.double_conv2 = double_conv(self.size[1], 256)
 self.up_conv3 = up_conv(256, 128)
 self.Att3 = Attention_block(F_g=128,F_l=152-128,F_int=16)
 self.double_conv3 = double_conv(self.size[2], 128)
 self.up_conv4 = up_conv(128, 64)
 self.Att4 = Attention_block(F_g=64,F_l=80-64,F_int=12)
 self.double_conv4 = double_conv(self.size[3], 64)
 if self.concat_input:
self.up_conv_input = up_conv(64, 32)
 self.double_conv_input = double_conv(self.size[4], 32)
 self.final_conv = nn.Conv2d(self.size[5], out_channels, kernel_size=1)
 @property
def n_channels(self):
n_channels_dict = {'efficientnet-b0': 1280, 'efficientnet-b1': 1280, 'efficientnet-b2': 1408,
'efficientnet-b3': 1536, 'efficientnet-b4': 1792, 'efficientnet-b5': 2048,
'efficientnet-b6': 2304, 'efficientnet-b7': 2560}
return n_channels_dict[self.encoder.name]
@property
def size(self):
size_dict = {'efficientnet-b0': [592, 296, 152, 80, 35, 32], 'efficientnet-b1': [592, 296, 152, 80, 35, 32],
'efficientnet-b2': [600, 304, 152, 80, 35, 32], 'efficientnet-b3': [608, 304, 160, 88, 35, 32],
'efficientnet-b4': [624, 312, 160, 88, 35, 32], 'efficientnet-b5': [640, 320, 168, 88, 35, 32],
'efficientnet-b6': [656, 328, 168, 96, 35, 32], 'efficientnet-b7': [672, 336, 176, 96, 35, 32]}
return size_dict[self.encoder.name]
def forward(self, x):
input_ = x
blocks = get_blocks_to_be_concat(self.encoder, x)
 _, x = blocks.popitem()
x1 = self.up_conv1(x)
 _, y = blocks.popitem()
x = self.Att1(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv1(x)
 x1 = self.up_conv2(x)
 y = blocks.popitem() [1]
x = self.Att2(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv2(x)
 x1 = self.up_conv3(x)
 y = blocks.popitem() [1]
x = self.Att3(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv3(x)
 x1 = self.up_conv4(x)
 y = blocks.popitem() [1]
x = self.Att4(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv4(x)
 if self.concat_input:
x = self.up_conv_input(x)
 x = torch.cat([x, input_], dim=1)
 x = self.double_conv_input(x)
 x = self.final_conv(x)
 return x
def get_efficientunet_b0(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b0', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b1(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b1', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b2(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b2', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b3(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b3', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b4(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b4', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b5(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b5', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b6(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b6', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b7(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b7', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
!g++ -pthread --std=c++17 -Ofast -o main main.cpp<define_variables>
unet_b0 = get_efficientunet_b0(out_channels=4, concat_input=True, pretrained=False)
 unet_b1 = get_efficientunet_b1(out_channels=4, concat_input=True, pretrained=False)
 FPN_res34 = smp.FPN('resnet34', classes=4, activation=None,encoder_weights=None)
 PSP_seresx50 = smp.PSPNet('se_resnext50_32x4d', classes=4, activation=None,encoder_weights=None)
 FPN_seresx50 = smp.FPN('se_resnext50_32x4d', classes=4, activation=None,encoder_weights=None)
 FPN_b0 = smp.FPN('efficientnet-b0', classes=4, activation=None,encoder_weights=None)
 FPN_b1 = smp.FPN('efficientnet-b1', classes=4, activation=None,encoder_weights=None)
 FPN_b2 = smp.FPN('efficientnet-b2', classes=4, activation=None,encoder_weights=None)
 FPN_b3 = smp.FPN('efficientnet-b3', classes=4, activation=None,encoder_weights=None)
 FPN_b4 = smp.FPN('efficientnet-b4', classes=4, activation=None,encoder_weights=None )
%%bash
time./main.. /input/frunk-optimization/final_frunkopt_1515566.8585.csv./submission.csv<load_from_csv>
def weight_loader(model,path):
ckpt_path = path
device = torch.device("cuda")
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"])
 return model
def make_eval(models):
for i,m in enumerate(models):
models[i]=m.eval()
return models
unet_b0 = weight_loader(unet_b0,'.. /input/b0fulldata/b0att_complete.pth')
 FPN_seresx50 = weight_loader(FPN_seresx50,'.. /input/b0fulldata/fpnserex50complete.pth')
 FPN_b1 = weight_loader(FPN_b1,'.. /input/fpn-b1/fpn_b1-stage-4.pth')
 FPN_b0 = weight_loader(FPN_b0,'.. /input/b0fulldata/fpn_b0-complete-stage-2.pth')
 unet_b1 = weight_loader(unet_b1,'.. /input/classres50/effb1unetbceloss_lrreduce_full_2.pth')
 unet_se_resnext50_32x4d = torch.jit.load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda().eval()
unet_resnet34 = torch.jit.load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda().eval()
FPN_b2 = weight_loader(FPN_b2,'.. /input/ubambamodels/fpn_b2-complete.pth')
 FPN_b4 = weight_loader(FPN_b4,'.. /input/fpnb41024/fpn-b4-crop-stage-5-2-all.pth')
 FPN_b3 = weight_loader(FPN_b3,'.. /input/fpnb3stage4/fpn-b3_crop-stage-4-all.pth' )
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv')
 cities_numb = len(cities)
 subm = pd.read_csv('.. /input/reversing-and-shifting/submission.csv' )<define_variables>
class Model_sig:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
x = torch.flip(x,dims = [-1])
 with torch.no_grad() :
for m in self.models:
flipped_mask = m(x)
 mask = torch.flip(flipped_mask, dims = [-1])
 res.append(mask)
 res =(torch.stack(res))
return torch.mean(torch.sigmoid(res), dim=0)
 class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
x = torch.flip(x,dims = [-1])
 with torch.no_grad() :
for m in self.models:
flipped_mask = m(x)
 mask = torch.flip(flipped_mask,dims = [-1])
 res.append(mask)
 res = torch.stack(res)
 return torch.sigmoid(torch.mean(res, dim=0))
model = Model([FPN_b3,unet_b0,unet_b1, FPN_seresx50, FPN_b0, FPN_b1, FPN_b2, unet_se_resnext50_32x4d, unet_resnet34,FPN_b4])
list_submits = [x for x in os.listdir(".. /input/")if x not in ['traveling-santa-2018-prime-paths', 'reversing-and-shifting']]<load_from_csv>
thresholds_max=[0.7,0.7,0.7,0.7]
thresholds_min=[0.2,0.2,0.3,0.3]
min_area=[350, 500, 750, 1000]
res = []
predictions = []
fnames_all = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = model(images.to(device)).detach().cpu().numpy()
for p, file in zip(batch_preds, fnames):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
p_channel_ = p_channel
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds_max[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 else:
p_channel =(p_channel_>thresholds_min[i] ).astype(np.uint8)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res )
filenames = []
for s in list_submits:
for ss in os.listdir(".. /input/"+ s):
if ss[-4:] == '.csv':
check = pd.read_csv(".. /input/"+ s + '/' + ss)
 if 'Path' in check.columns:
filenames += [".. /input/"+ s + '/' + ss]<feature_engineering>
df.to_csv("submission.csv", index=False )
path_dict = {}
sub_perm_dict = {}
for file_name in filenames:
path_dict[file_name] = pd.read_csv(file_name ).Path.tolist()
sub_perm_dict[file_name] = np.zeros(len(path_dict[file_name]))
for j, x in enumerate(path_dict[file_name][:-1]):
sub_perm_dict[file_name][x] = j<compute_test_metric>
def change(x):
x=x[-1:]
return x
df_=df.copy()
df_.ImageId_ClassId=df_.ImageId_ClassId.apply(change)
 df_[df_.EncodedPixels!=''].ImageId_ClassId.value_counts()
def get_dist(x1, y1, x2, y2):
return np.sqrt(( x1 - x2)** 2 +(y1 - y2)** 2 )<prepare_x_and_y>
city_X = np.zeros(cities_numb)
 city_Y = np.zeros(cities_numb)
 for city, x, y in zip(cities.CityId.tolist() , cities.X.tolist() , cities.Y.tolist()):
city_X[city] = x
city_Y[city] = y<load_pretrained>