Kaggle competition: Traveling Santa 2018 - Prime Paths
<SOS> metric: TravelingSanta2 Kaggle data source: traveling-santa-2018-prime-paths<choose_model_class>
import numba
import numpy as np
import pandas as pd
from math import sqrt
from functools import lru_cache
from sklearn.neighbors import KDTree
from sympy import isprime, primerange
from tqdm import tqdm_notebook as tqdm
from ortools.constraint_solver import pywrapcp
from itertools import combinations, permutations
from scipy.spatial.distance import pdist, squareform
from ortools.constraint_solver import routing_enums_pb2
from sympy.utilities.iterables import multiset_permutations
class AdamW(Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.init_lr = lr
self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
eta_t = lr / self.init_lr
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- eta_t * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<define_search_model>
cities = pd.read_csv('.. /input/reeindeer/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )
glorotInit = initializers.glorot_uniform(seed=RANDOM_STATE)
 orthoInit = initializers.Orthogonal(seed=RANDOM_STATE )<import_modules>
kdt = KDTree(XY )
def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel)
 class Attention(Layer):
def __init__(self,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True,
return_attention=False,
**kwargs):

 self.supports_masking = True
self.return_attention = return_attention
self.init = glorotInit
self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
eij = dot_product(x, self.W)
 if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 weighted_input = x * K.expand_dims(a)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, a]
return result
def compute_output_shape(self, input_shape):
if self.return_attention:
return [(input_shape[0], input_shape[-1]), 
(input_shape[0], input_shape[1])]
else:
return input_shape[0], input_shape[-1]<compute_test_metric>
path = np.array(pd.read_csv('.. /input/reeindeer/path1515413.csv' ).Path)
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<define_search_model>
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path)
 @numba.jit
def chunk_scores(chunk):
scores = np.zeros(10)
 pure_distance = 0
for i in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[i], chunk[i+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if is_not_prime[id_from]:
scores[9-i%10] += distance
scores *= 0.1
scores += pure_distance
return scores
@numba.jit('f8(i8, i8, i8[:], i8[:], i8[:], i8, f8[:,:], i8[:])', nopython=True, parallel=False)
 def score_compound_chunk(offset, head, firsts, lasts, lens, tail, scores, indexes):
score = 0.0
last_city_id = head
for i in numba.prange(len(indexes)) :
index = indexes[i]
first, last, chunk_len = firsts[index], lasts[index], lens[index]
score += cities_distance(offset, last_city_id, first)
 score += scores[index,(offset + 1)% 10]
last_city_id = last
offset += chunk_len
return score + cities_distance(offset, last_city_id, tail)
 @numba.jit('i8(i8, i8, i8[:], i8[:], i8[:], i8, f8[:,:], i8[:,:], f8)', nopython=True, parallel=False)
 def best_score_permutation_index(offset, head, firsts, lasts, lens, tail, scores, indexes, best_score):
best_index = -1
for i in numba.prange(len(indexes)) :
score = score_compound_chunk(offset, head, firsts, lasts, lens, tail, scores, indexes[i])
 if score < best_score:
best_index, best_score = i, score
return best_index
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def sum_distance(ids):
res = 0
for i in numba.prange(len(ids)) :
for j in numba.prange(i + 1, len(ids)) :
res += cities_distance(0, ids[i], ids[j])
 return res
@lru_cache(maxsize=None)
 def indexes_permutations(n):
return np.array(list(map(list, permutations(range(n)))) )
glorotInit = initializers.glorot_uniform(seed=RANDOM_STATE)
 orthoInit = initializers.Orthogonal(seed=RANDOM_STATE )<choose_model_class>
trees = get_uplets(3, 5, 5 )
def srnn_model() :
embedding_layer = Embedding(NUM_WORDS,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=16,
trainable=False)
 input1 = Input(shape=(16,), dtype='int32')
 embed = embedding_layer(input1)
 drop1 = SpatialDropout1D(0.2 )(embed)
 gru1 = Bidirectional(CuDNNLSTM(NUM_FILTERS, return_sequences=True, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(drop1)
 gru1_2 = Bidirectional(CuDNNLSTM(NUM_FILTERS, return_sequences=True, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(gru1)
 atten_1 = Attention()(gru1)
 atten_1_2 = Attention()(gru1_2)
 max_pool1 = GlobalMaxPooling1D()(gru1_2)
 avg_pool1 = GlobalAveragePooling1D()(gru1_2)
 conc1 = concatenate([atten_1, atten_1_2, max_pool1, avg_pool1])
 Encoder1 = Model(input1, conc1)
 input2 = Input(shape=(2, 16,), dtype='int32')
 embed2 = TimeDistributed(Encoder1 )(input2)
 gru2 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(embed2)
 Encoder2 = Model(input2, gru2)
 input3 = Input(shape=(2, 2, 16), dtype='int32')
 embed3 = TimeDistributed(Encoder2 )(input3)
 gru3 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False, kernel_initializer=glorotInit,
recurrent_initializer=glorotInit))(embed3)
 dense = Dense(32, activation="relu", kernel_initializer=glorotInit )(gru3)
 drop2 = Dropout(0.1 )(dense)
 output = Dense(1, activation="sigmoid" )(drop2)
 model = Model(input3, output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(lr=LR, weight_decay=0.) , metrics=["acc", f1])
 return model<choose_model_class>
path_index = np.argsort(path[:-1])
 print(f'Total score is {score_path(path):.2f}.')
 for ids in tqdm(trees):
i1, i2, i3 = np.sort(path_index[ids])
 head, tail = path[i1-1], path[i3+1]
all_chunks = [[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1]]]
for i, chunks in enumerate(all_chunks):
chunks = [chunk for chunk in chunks if len(chunk)]
scores = np.array([chunk_scores(chunk)for chunk in chunks])
 lens = np.array([len(chunk)for chunk in chunks])
 firsts = np.array([chunk[0] for chunk in chunks])
 lasts = np.array([chunk[-1] for chunk in chunks])
 if i == 0:
best_score = score_compound_chunk(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) [0])
 index = best_score_permutation_index(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) , best_score)
 if index > 0:
perm = [chunks[i] for i in indexes_permutations(len(chunks)) [index]]
path[i1-1:i3+2] = np.concatenate([[head], np.concatenate(perm), [tail]])
 path_index = np.argsort(path[:-1])
 print(f'New total score is {score_path(path):.3f}.Permutating path at indexes {i1}, {i2}, {i3}.')
 break
break
<choose_model_class>
fours = get_uplets(4, 5, 0 )
clr = CyclicLR(base_lr=0.001, max_lr=0.004,
step_size=300., mode='exp_range',
gamma=0.99994 )<train_model>
path_index = np.argsort(path[:-1])
 print(f'Total score is {score_path(path):.2f}.')
 for ids in tqdm(fours):
i1, i2, i3, i4 = np.sort(path_index[ids])
 head, tail = path[i1-1], path[i4+1]
all_chunks = [[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]]
]
for i, chunks in enumerate(all_chunks):
chunks = [chunk for chunk in chunks if len(chunk)]
scores = np.array([chunk_scores(chunk)for chunk in chunks])
 lens = np.array([len(chunk)for chunk in chunks])
 firsts = np.array([chunk[0] for chunk in chunks])
 lasts = np.array([chunk[-1] for chunk in chunks])
 if i == 0:
best_score = score_compound_chunk(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) [0])
 index = best_score_permutation_index(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) , best_score)
 if index > 0:
perm = [chunks[i] for i in indexes_permutations(len(chunks)) [index]]
path[i1-1:i4+2] = np.concatenate([[head], np.concatenate(perm), [tail]])
 path_index = np.argsort(path[:-1])
 print(f'New total score is {score_path(path):.3f}.Permutating path at indexes {i1}, {i2}, {i3}, {i4}.')
 break
break
def train_pred(model, train_X, train_y, val_X, val_y, test_data, callbacks=None):
st = time.time()
model.fit(np.array(train_X), train_y, batch_size=BATCH_SIZE, epochs=NUM_EPOCH, validation_data=(np.array(val_X), val_y),
callbacks = callbacks, verbose=2)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = threshold_search(val_y, pred_val_y)
 print("\tVal F1 Score: {:.4f}\tThresh: {:.2f}".format(best_thresh["f1"], best_thresh["threshold"]))
best_score = best_thresh["f1"]
pred_test_y = model.predict(test_data, batch_size=1024, verbose=0)
 print("Training time was: " + str(time.time() - st))
print('=' * 60)
 return pred_val_y, pred_test_y, best_score<split>
num_iters = 300
path_df = pd.DataFrame({"Path": path})
 cities2 = cities.reset_index().rename(columns={"CityId": "Path"})
 path_df = path_df.merge(cities2,how='left',on='Path')
 pnums = list(primerange(0, path_df.shape[0]))
status_dict = {0: 'ROUTING_NOT_SOLVED',
1: 'ROUTING_SUCCESS',
2: 'ROUTING_FAIL',
3: 'ROUTING_FAIL_TIMEOUT',
4: 'ROUTING_INVALID'}
def create_mat(df):
mat = pdist(df)
 return squareform(mat)
 def create_distance_callback(dist_matrix):
def distance_callback(from_node, to_node):
return int(dist_matrix[from_node][to_node])
 return distance_callback
def optimize(df, startnode, stopnode, fixed):
num_nodes = df.shape[0]
mat = create_mat(df)
 dist_callback = create_distance_callback(mat)
 search_parameters = pywrapcp.RoutingModel.DefaultSearchParameters()
search_parameters.solution_limit = num_iters
search_parameters.first_solution_strategy =(
 routing_enums_pb2.FirstSolutionStrategy.LOCAL_CHEAPEST_INSERTION)
 search_parameters.local_search_metaheuristic =(
 routing_enums_pb2.LocalSearchMetaheuristic.GUIDED_LOCAL_SEARCH)
 if fixed:
routemodel = pywrapcp.RoutingModel(num_nodes, 1, [startnode], [stopnode])
 else:
routemodel = pywrapcp.RoutingModel(num_nodes, 1, startnode)
 routemodel.SetArcCostEvaluatorOfAllVehicles(dist_callback)
 assignment = routemodel.SolveWithParameters(search_parameters)
 return routemodel, assignment
def get_route(df, startnode, stopnode, fixed):
routemodel, assignment = optimize(df, int(startnode), int(stopnode), fixed)
 route_number = 0
node = routemodel.Start(route_number)
 route = []
while not routemodel.IsEnd(node):
route.append(node)
 node = assignment.Value(routemodel.NextVar(node))
return route
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(len(x_test_padded_seqs_split))
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=RANDOM_STATE ).split(x_train_padded_seqs_split, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train_tmp = x_train_padded_seqs_split[train_idx]
y_train_tmp = y_train[train_idx]
X_val_tmp = x_train_padded_seqs_split[valid_idx]
y_val_tmp = y_train[valid_idx]
model = srnn_model()
pred_val_y, pred_test_y, best_score = train_pred(model, X_train_tmp, y_train_tmp, X_val_tmp, y_val_tmp, x_test_padded_seqs_split,
callbacks=[clr])
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits)
 del(X_train_tmp)
 del(y_train_tmp)
 del(X_val_tmp)
 del(y_val_tmp)
 gc.collect()<save_to_csv>
def run_opt(df, m):
print(f'm = {m}')
 n = int(path_df.shape[0] / m)* 2
for i in tqdm(range(n)) :
startpoint = int(1 + i *(df.shape[0] - m - 2)/ n)
 endpoint = min(( startpoint + m), df.shape[0])
 district = df.iloc[startpoint:endpoint, :3].copy()
district = district.reset_index()
locations = district[['X', 'Y']].values
segnodes = get_route(locations, 0,(m-1), fixed=True)
 ord_district = district.iloc[segnodes]
segment = ord_district.index.tolist()
temp = district.loc[segment, ['Path','X', 'Y']].reset_index()
district_2 = district.copy()
district_2.iloc[:(m-1),1:] = temp.copy()
district = district.set_index('index')
 district_2 = district_2.set_index('index')
 district['step'] = np.sqrt(( district.X - district.X.shift())**2 +(district.Y - district.Y.shift())**2)
 district['step_adj'] = np.where(( district.index)% 10 != 0, district.step, district.step +
district.step*0.1*(~district.Path.shift().isin(pnums)))
 district_2['step'] = np.sqrt(( district_2.X - district_2.X.shift())**2 +(district_2.Y - district_2.Y.shift())**2)
 district_2['step_adj'] = np.where(( district_2.index)% 10 != 0, district_2.step, district_2.step +
district_2.step*0.1*(~district_2.Path.shift().isin(pnums)))
 check_dist = district.step_adj.sum() > district_2.step_adj.sum()
if check_dist:
df.iloc[startpoint:endpoint,0:3] = district_2
print(f"Improved of {district.step_adj.sum() - district_2.step_adj.sum() }")
 break
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )<feature_engineering>
m = 40
run_opt(path_df, m)
 path = path_df['Path'].values
( time.time() - startTimePt2)/ 60 
<import_modules>
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )
<choose_model_class><EOS> 
make_submission("path" + str(int(score_path(path))), path )