Kaggle competition: Traveling Santa 2018 - Prime Paths
del(train)
 del(test)
 gc.collect() 
<load_pretrained>
For the Traveling Santa 2018 competition.All computation is done
within the Kernel, including building the list of primes and creating the TSPLIB file.
 The downloaded C codes(in gzipped tar files)are left in the Kernel.
 Note: The prime_thread code, built on top of the linkern code from Concorde, uses as a
random seed the current time.The behavior of the Kernel is random .
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
NUM_WORDS = None
word_idx = build_vocab(df_texts["treated_question"], NUM_WORDS)
 NUM_WORDS = len(word_idx)+ 1
print("
****** LOAD EMBEDDINGS ******")
 print("
---Extracting GloVe embedding---")
 embed_glove = load_embed(glove)
 index_embs(embed_glove, word_idx, NUM_WORDS, fileName="glove")
 del(embed_glove)
 gc.collect()
---Extracting Paragram embedding---")
 print("
---Extracting FastText embedding---")
 embed_fasttext = load_embed(wiki_news)
 index_embs(embed_fasttext, word_idx, NUM_WORDS, fileName="fasttext")
 del(embed_fasttext)
 gc.collect()<set_options>
cities = pd.read_csv('.. /input/cities.csv', index_col=['CityId'], nrows=None)
 cities_1000 = cities * 1000
gc.collect()
%reset -f<import_modules>
def write_tsp(nodes, filename, name='Santa Prime Paths'):
with open(filename, 'w')as f:
f.write('NAME : %s
' % name)
 f.write('COMMENT : %s
' % name)
 f.write('TYPE : TSP
')
 f.write('DIMENSION : %d
' % len(nodes))
f.write('EDGE_WEIGHT_TYPE : EUC_2D
')
 f.write('NODE_COORD_SECTION
')
 for row in nodes.itertuples() :
f.write('%d %.11f %.11f
' %(row.Index + 1, row.X, row.Y))
f.write('EOF
')
 write_tsp(cities_1000, 'santa197769.tsp' )
startTimePt2 = time.time()<define_variables>
%%bash
rm -r LKH-2.0.9
rm LKH-2.0.9.t*
wget http://akira.ruc.dk/~keld/research/LKH/LKH-2.0.9.tgz
NUM_WORDS = None
EMBEDDING_DIM = 300
NUM_FILTERS = 50
MAX_LEN = 64
BATCH_SIZE = 2560
RANDOM_STATE = 42
NUM_EPOCH = 11
LR = 0.001
LR_MAX = LR * 6
WD = 0.011 *(BATCH_SIZE / 979591 / NUM_EPOCH)**0.5
STEP_SIZE_CLR = 2 *(979591 / BATCH_SIZE)
<define_variables>
%%bash
tar xzvf LKH-2.0.9.tgz
cd LKH-2.0.9
make
mv LKH..
cd..
rm -r LKH-2.0.9
np.random.seed(RANDOM_STATE)
 set_random_seed(RANDOM_STATE )<load_from_csv>
%%bash 
./LKH par0.par
df_train = pd.read_csv("train_processed_text.csv")
 df_test = pd.read_csv("test_processed_text.csv")
 print("Train shape : ", df_train.shape)
 print("Test shape : ", df_test.shape )<prepare_x_and_y>
%%bash 
./LKH par7.par
X_train = df_train["treated_question"].fillna("_na_" ).values
X_test = df_test["treated_question"].fillna("_na_" ).values
y_train = df_train['target'].values<load_pretrained>
def write_xy(nodes, filename):
with open(filename, 'w')as f:
f.write('%d
' % len(nodes))
for row in nodes.itertuples() :
f.write('%.12f %.12f
' %(row.X, row.Y))
f.write('EOF
')
 write_xy(cities, 'kaggle.xy' )
with open('tokenizer.pickle', 'rb')as handle:
tokenizer = pickle.load(handle )<string_transform>
%%bash
wget http://www.math.uwaterloo.ca/tsp/pm/gen_primes.c
gcc -o gen_primes gen_primes.c -lm 
./gen_primes
head primes.txt
rm gen_primes*
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 print("hello word" )<define_variables>
%%bash
wget http://www.math.uwaterloo.ca/tsp/pm/PM_1.tgz
tar xzvf PM_1.tgz
cd PM_1
make prime_thread
mv prime_thread..
cd..
rm -r PM_1
NUM_WORDS = len(tokenizer.word_index)
 NUM_WORDS<string_transform>
%%bash
rm -r PM-LKH*
wget http://www.math.uwaterloo.ca/tsp/pm/PM-LKH-3b.tgz
tar xzvf PM-LKH-3b.tgz
mv PM-LKH-3b PM-LKH
cd PM-LKH
make
rm -r PMSRC_DIV
X_train = pad_sequences(X_train, maxlen=MAX_LEN)
 X_test = pad_sequences(X_test, maxlen=MAX_LEN)
 print("hello word" )<drop_column>
%%bash
cp kaggle.xy PM-LKH/
cp santa197769.tsp PM-LKH/
cp primes.txt PM-LKH/primes_list
del(df_train)
 del(df_test)
 del(tokenizer )<load_pretrained>
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
rm -r DIV
rm -r DIV_TOURS
embedding_matrix_1 = np.load("glove.npy")
 embedding_matrix_2 = np.load("fasttext.npy")
 print("Done..." )<compute_test_metric>
%%bash
cd PM-LKH 
./run_Segment_Optimization post.tour 5000 post2.tour
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis = 0)
 np.shape(embedding_matrix )<drop_column>
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
rm -r DIV
rm -r DIV_TOURS
del(embedding_matrix_1)
 del(embedding_matrix_2)
<set_options>
%%bash
cd PM-LKH 
./run_Segment_Optimization post2.tour 7500 post3.tour
gc.collect()
print("done" )<split>
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
rm -r DIV
rm -r DIV_TOURS
def SliceDataSRNN(data):
slicedData = []
for i in range(data.shape[0]):
split1 = np.split(data[i], 2)
 a=[]
for j in range(2):
s=np.split(split1[j], 2)
 a.append(s)
 slicedData.append(a)
 arr = np.array(slicedData, dtype=np.int32)
 del(slicedData)
 del(a)
 del(s)
 del(split1)
 gc.collect()
return arr<randomize_order>
%%bash
cd PM-LKH 
./run_Segment_Optimization post3.tour 6250 post4.tour
x_train_padded_seqs_split = SliceDataSRNN(X_train)
 x_test_padded_seqs_split = SliceDataSRNN(X_test )<drop_column>
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
mv post4.tour.. /post4.tour
rm -r DIV
rm -r DIV_TOURS
<choose_model_class><EOS> 
Final run of the linkern-based code to search for global impprovemnts .