Kaggle competition: Traveling Santa 2018 - Prime Paths
<SOS> metric: TravelingSanta2 Kaggle data source: traveling-santa-2018-prime-paths<predict_on_test>
import numpy as np
import pandas as pd
import numba
from sympy import isprime, primerange
from math import sqrt
from sklearn.neighbors import KDTree
from tqdm import tqdm
from itertools import combinations
del glove_dic
gc.collect()
model.load_weights('./{}.hdf5'.format(model_name))
y_pred_glove = model.predict(q_val,batch_size=1064,verbose=1)
 pred_glove = model.predict(q_test,batch_size=1064,verbose=1)
<feature_engineering>
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )
emb_file = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
emb_dic = {}
for line in tqdm_notebook(open(emb_file,encoding='utf-8',errors='ignore')) :
temp = line.split(" ")
 emb_dic[temp[0]] = np.asarray(temp[1:],dtype='float32')
 emb_matrix = np.zeros(( n_words,emb_size))
for w,index in word_index.items() :
if index >= n_words:
continue
vec = emb_dic.get(w)
 if vec is not None:
emb_matrix[index,:] = vec<choose_model_class>
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path )
inp = Input(shape=(max_len,))
x = Embedding(input_dim=n_words,output_dim=emb_size, weights=[emb_matrix],trainable=False )(inp)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = GlobalAveragePooling1D()(x1)
 x2 = GlobalAveragePooling1D()(x2)
 concat = concatenate([x1, x2])
 x = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp,output=x)
 model.summary()
model_name = 'lstm_paragram_emb'
checkpoint = ModelCheckpoint(filepath='./{}.hdf5'.format(model_name),
monitor='val_loss',mode='min',verbose=1,
save_best_only=True )<train_model>
kdt = KDTree(XY )
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc',f1])
 history = model.fit(q_train,y_train,batch_size=1500,epochs=5,
validation_data=(q_val,y_val),verbose=1,callbacks=[checkpoint] )<predict_on_test>
path = np.array(pd.read_csv('.. /input/traveling-santa-lkh-solution/pure1502650.csv' ).Path)
 initial_score = score_path(path )
del emb_dic
gc.collect()
model.load_weights('./{}.hdf5'.format(model_name))
y_pred_paragram = model.predict(q_val,batch_size=1064,verbose=1)
 pred_paragram = model.predict(q_test,batch_size=1064,verbose=1)
<feature_engineering>
path_index = np.argsort(path[:-1])
 total_score = initial_score
print(f'Total score is {total_score:.2f}.')
 for _ in range(3):
for step,(id1, id2)in enumerate(tqdm(pairs), 1):
if step % 10**6 == 0:
new_total_score = score_path(path)
 print(f'Score: {new_total_score:.2f}; improvement over last 10^6 steps: {total_score - new_total_score:.2f}; total improvement: {initial_score - new_total_score:.2f}.')
 total_score = new_total_score
i, j = path_index[id1], path_index[id2]
i, j = min(i, j), max(i, j)
 chunk, reversed_chunk = path[i-1:j+2], np.concatenate([path[i-1:i], path[j:i-1:-1], path[j+1:j+2]])
 chunk_score, reversed_chunk_score = score_chunk(i-1, chunk), score_chunk(i-1, reversed_chunk)
 if j - i > 2:
chunk_abc = np.concatenate([path[i-1:i+1], path[j:i:-1], path[j+1:j+2]])
 chunk_acb = np.concatenate([path[i-1:i], path[j:j+1], path[i:j], path[j+1:j+2]])
 chunk_abcb = np.concatenate([path[i-1:i+1], path[j:j+1], path[i+1:j], path[j+1:j+2]])
 abc_score, acb_score, abcb_score = map(lambda chunk: score_chunk(i-1, chunk), [chunk_abc, chunk_acb, chunk_abcb])
 for chunk, score, name in zip(( chunk_abc, chunk_acb, chunk_abcb),(abc_score, acb_score, abcb_score),('abc', 'acb', 'abcb')) :
if score < chunk_score:
path[i-1:j+2] = chunk
path_index = np.argsort(path[:-1])
 chunk_score = score
if reversed_chunk_score < chunk_score:
path[i-1:j+2] = reversed_chunk
path_index = np.argsort(path[:-1] )
emb_file = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
emb_dic = {}
for line in tqdm_notebook(open(emb_file,encoding='utf-8',errors='ignore')) :
temp = line.split(" ")
 emb_dic[temp[0]] = np.asarray(temp[1:],dtype='float32')
 emb_matrix = np.zeros(( n_words,emb_size))
for w,index in word_index.items() :
if index >= n_words:
continue
vec = emb_dic.get(w)
 if vec is not None:
emb_matrix[index,:] = vec<choose_model_class>
print(f'Total improvement is {initial_score - total_score:.2f}.' )
inp = Input(shape=(max_len,))
x = Embedding(input_dim=n_words,output_dim=emb_size, weights=[emb_matrix] )(inp)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = GlobalAveragePooling1D()(x1)
 x2 = GlobalAveragePooling1D()(x2)
 concat = concatenate([x1, x2])
 x = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp,output=x)
 model.summary()
model_name = 'lstm_wiki_emb'
checkpoint = ModelCheckpoint(filepath='./{}.hdf5'.format(model_name),
monitor='val_loss',mode='min',verbose=1,
save_best_only=True )<train_model>
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )
<predict_on_test><EOS> 
make_submission(score_path(path), path )