Kaggle competition: Traveling Santa 2018 - Prime Paths
<SOS> metric: TravelingSanta2 Kaggle data source: traveling-santa-2018-prime-paths<compute_train_metric>
import numpy as np
import pandas as pd
import time
import math
import numba
import sympy
from sympy.utilities.iterables import multiset_permutations
def calculate_overall_lwlrap_sklearn(truth, scores):

 sample_weight = np.sum(truth > 0, axis=1)
 nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)
 overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(
 truth[nonzero_weight_sample_indices, :] > 0,
scores[nonzero_weight_sample_indices, :],
sample_weight=sample_weight[nonzero_weight_sample_indices])
 return overall_lwlrap<choose_model_class>
def load_cities(filename):
cities = pd.read_csv(filename)
 city_id = cities.CityId.astype(np.int32)
 loc = np.vstack([cities.X.astype(np.float32), cities.Y.astype(np.float32)] ).transpose()
is_prime = np.array([1 if sympy.isprime(i)else 0 for i in city_id], dtype=np.int32)
 return(city_id, loc, is_prime)
 def load_tour(filename):
tour = pd.read_csv(filename)
 tour = tour.Path.values.astype(np.int32)
 return tour
GlobalMaxPooling2D, MaxPooling2D, concatenate, Activation, ZeroPadding2D, Dropout, add, ReLU)
 def cnn_6(lr, do=0, l2reg=0):
inp = Input(shape=(64,128,1))
x = BatchNormalization()(inp)
 x = ReLU()(x)
 x = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same' )(x)
 x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='same' )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 x = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same' )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 x = Conv2D(filters=64, kernel_size=(3,3), strides=(2,2), kernel_regularizer=l2(l2reg), padding='same' )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 x = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same' )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 x = Conv2D(filters=128, kernel_size=(3,3), strides=(2,2), kernel_regularizer=l2(l2reg), padding='same' )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 x = Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), kernel_regularizer=l2(l2reg), padding='same' )(x)
 x = GlobalMaxPooling2D()(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(do )(x)
 out = Dense(80, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=out)
 opt = Adam(lr=lr)
 model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=[])
 return model<define_variables>
def save_tour(filename, tour):
with open(filename, "w")as f:
f.write("Path
")
 for i in tour:
f.write(str(i))
f.write("
" )
SAMPLE_DURATION = 3.0
SAMPLE_RATE = 44100
L = int(SAMPLE_RATE * SAMPLE_DURATION)
 def random_slice(samples):
begin = np.random.randint(0, len(samples)- L)
 return samples[begin : begin+L]
def pad(samples):
if len(samples)>= L:
return samples
else:
return np.pad(samples, pad_width=(L - len(samples), 0), mode='constant', constant_values=(0, 0))
training_data = []
progress = 0
for file_name in os.listdir('.. /input/freesound-audio-tagging-2019/train_curated/'):
file_data, _ = librosa.core.load('.. /input/freesound-audio-tagging-2019/train_curated/' + file_name, sr=SAMPLE_RATE)
 duration = librosa.core.get_duration(file_data, sr=SAMPLE_RATE)
 for i in range(math.ceil(duration/SAMPLE_DURATION)) :
begin = i*L
file_data_i = file_data[begin:begin+L]
if len(file_data_i)< L:
file_data_i = pad(file_data_i)
 file_array_i = librosa.feature.mfcc(file_data_i,
sr=SAMPLE_RATE,
n_fft=2560,
hop_length=347*int(SAMPLE_DURATION),
n_mels=128,
n_mfcc=64)
 training_data.append(( file_name+'_'+str(i), file_array_i, train_labels_dict[file_name]))
if progress%100 == 0:
print('processed', progress, 'files')
 progress += 1
print('train data done')
 random.Random(1001 ).shuffle(training_data)
 test = pd.read_csv(".. /input/freesound-audio-tagging-2019/sample_submission.csv")
 test_data = []
progress = 0
for file_name in test['fname'].tolist() :
file_data, _ = librosa.core.load('.. /input/freesound-audio-tagging-2019/test/' + file_name, sr=SAMPLE_RATE)
 duration = librosa.core.get_duration(file_data, sr=SAMPLE_RATE)
 for i in range(math.ceil(duration/SAMPLE_DURATION)) :
begin = i*L
file_data_i = file_data[begin:begin+L]
if len(file_data_i)< L:
file_data_i = pad(file_data_i)
 file_array_i = librosa.feature.mfcc(file_data_i,
sr=SAMPLE_RATE,
n_fft=2560,
hop_length=347*int(SAMPLE_DURATION),
n_mels=128,
n_mfcc=64)
 test_data.append(( file_name+'_'+str(i), file_array_i))
if progress%100 == 0:
print('processed', progress, 'files')
 progress += 1
print('test data done' )<train_model>
@numba.jit('f4(f4[:], f4[:])', nopython=True)
 def euc_2d(a, b):
xd = a[0] - b[0]
yd = a[1] - b[1]
return math.sqrt(xd * xd + yd * yd)
 @numba.jit('f8(i4[:], f4[:,:], i4[:])', nopython=True, parallel=True)
 def cost_santa2018(tour, loc, is_prime):
dist = 0.0
for i in numba.prange(1, tour.shape[0]):
a = tour[i - 1]
b = tour[i]
d = euc_2d(loc[a], loc[b])
 if i % 10 == 0 and is_prime[a] == 0:
d *= 1.1
dist += d
return dist
def step_decay(epoch):
initial_lrate = 4e-4
drop = 0.5
epoch_cutoff = 7
epochs_drop = 3.0
if epoch<epoch_cutoff:
return initial_lrate
else:
return initial_lrate * math.pow(drop, math.floor(( 1+epoch-epoch_cutoff)/epochs_drop))
train_files = all_files
X_train, y_train = get_train(train_files)
 X_train = np.expand_dims(X_train, axis=3)
 filenames_test, X_test = get_test()
X_test = np.expand_dims(X_test, axis=3)
 model = cnn_6(lr=4e-4, do=0.5, l2reg=0)
 model.summary()
lrate = LearningRateScheduler(step_decay, verbose=1)
 model_history = model.fit(X_train, y_train, batch_size=128, epochs=20, verbose=1, callbacks=[lrate])
 y_train_pred = model.predict(X_train, batch_size=128)
 train_lwlwrap = calculate_overall_lwlrap_sklearn(y_train, y_train_pred)
 print('train lwlwrap score:', train_lwlwrap)
 y_test_pred = model.predict(X_test, batch_size=128)
 filnames_test_parents, y_test_pred, _ = geometric_average_val_scores(filenames_test, y_test_pred, y_test_pred)
 print('training complete' )<save_to_csv>
def gen_subprobs(n, k):
return np.array([[i, i + k] for i in range(n - k)], dtype=np.int32)
 def gen_perms(k):
idx = np.arange(1, k - 1, dtype=np.int32)
 perm = []
for p in multiset_permutations(idx):
perm.append(np.hstack([[0], p, [k-1]]))
return np.array(perm, dtype=np.int32)
 @numba.jit('f4(i4[:], f4[:,:], i4[:], i4[:])', nopython=True)
 def subprob_cost(tour, dist_mat, s_10th, s_prime):
dist = 0.0
for t in range(1, tour.shape[0]):
i = tour[t - 1]
j = tour[t]
d = dist_mat[i][j]
if s_10th[t] != 0 and s_prime[i] != 0:
d *= 1.1
dist += d
return dist
@numba.jit('i4(f4[:], i4[:,:,:], i8, i4[:,:], i4[:], f4[:,:], i4[:], i4[:,:])', nopython=True, parallel=True)
 def finetune_(best_improvement, best_perm, k, subprobs, tour, loc, is_prime, perm):
updated = 0
init_tour = np.arange(k ).astype(np.int32)
 for t in numba.prange(subprobs.shape[0]):
idx = np.arange(subprobs[t][0], subprobs[t][1] ).astype(np.int32)
 s_loc = loc[tour[idx]]
s_prime =(1 - is_prime[tour[idx]] ).astype(np.int32)
 s_10th = np.empty(( k,), dtype=np.int32)
 s_dist_mat = np.empty(( k, k), dtype=np.float32)
 for i in range(k):
s_10th[i] = 1 if(i + idx[0])% 10 == 0 else 0
for i in range(k):
for j in range(i + 1, k):
s_dist_mat[i][j] = s_dist_mat[j][i] = euc_2d(s_loc[i], s_loc[j])
 costs = np.empty(( perm.shape[0],), dtype=np.float32)
 init_cost = subprob_cost(init_tour, s_dist_mat, s_10th, s_prime)
 for i in range(perm.shape[0]):
costs[i] = subprob_cost(perm[i], s_dist_mat, s_10th, s_prime)
 best_index = np.argmin(costs)
 delta = init_cost - costs[best_index]
if delta > best_improvement[t]:
best_perm[t][0] = idx
best_perm[t][1] = idx[perm[best_index]]
best_improvement[t] = delta
updated += 1
return updated
def finetune(k, subprobs, tour, loc, is_prime, perm):
updated = 0
best_perm = np.empty(( subprobs.shape[0], 2, k), dtype=np.int32)
 best_improvement = np.zeros(( subprobs.shape[0],), dtype=np.float32)
 finetune_(best_improvement, best_perm, k, subprobs, tour, loc, is_prime, perm)
 used = []
updates = np.argsort(best_improvement)[::-1]
for i in updates:
delta = best_improvement[i]
if delta > 0:
s = best_perm[i][0][0]
e = best_perm[i][0][-1]
if any([(r[0] < s and s < r[1])or(r[0] < e and e < r[1])for r in used]):
continue
used.append(( s, e))
tour[best_perm[i][0]] = tour[best_perm[i][1]]
updated += 1
else:
break
return updated
predictions = pd.DataFrame(data=y_test_pred)
 predictions.insert(loc=0, column='fname', value=filnames_test_parents)
 columns = ['fname']
columns.extend(mlb.classes_.tolist())
 predictions.columns = columns
predictions.to_csv('submission.csv', index=False )<load_from_csv>
city_id, loc, is_prime = load_cities(".. /input/traveling-santa-2018-prime-paths/cities.csv")
 tour = load_tour(".. /input/dataset/submission.csv")
 init_cost = cost_santa2018(tour, loc, is_prime)
 print("initial cost", init_cost )
<normalization><EOS> 
K = 10 + 2
subprobs = gen_subprobs(city_id.shape[0], K)
 subprob_perms = gen_perms(K)
 print("** perm={}!".format(K-2))
t = time.time()
updated = finetune(K, subprobs, tour, loc, is_prime, subprob_perms)
 cost = cost_santa2018(tour, loc, is_prime)
 print("cost: {:.4f}, improvement: {:.4f}, found: {}, time: {:.2f}".format(cost, init_cost - cost, updated, time.time() - t))
save_tour("submission.csv", tour )