Kaggle competition: Traveling Santa 2018 - Prime Paths
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1 )<find_best_params>
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv')
 cities_numb = len(cities)
 subm = pd.read_csv('.. /input/reversing-and-shifting/submission.csv' )
max_f1 = 0
for thresh in np.arange(0.2, 0.601, 0.01):
thresh = np.round(thresh, 2)
 current_f1 = metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))
print("ค่า F1 ที่ threshold %.2f is %.4f" %(thresh, current_f1))
if current_f1 > max_f1:
best_thresh = thresh
max_f1 = current_f1
print('ค่า threshold ที่ดีที่สุดคือ %.2f ให้ค่า F1 เท่ากับ %.4f' %(best_thresh,max_f1))<predict_on_test>
list_submits = [x for x in os.listdir(".. /input/")if x not in ['traveling-santa-2018-prime-paths', 'reversing-and-shifting']]
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<import_modules>
filenames = []
for s in list_submits:
for ss in os.listdir(".. /input/"+ s):
if ss[-4:] == '.csv':
check = pd.read_csv(".. /input/"+ s + '/' + ss)
 if 'Path' in check.columns:
filenames += [".. /input/"+ s + '/' + ss]
del embedding_matrix, model, x
time.sleep(10 )<define_variables>
path_dict = {}
sub_perm_dict = {}
for file_name in filenames:
path_dict[file_name] = pd.read_csv(file_name ).Path.tolist()
sub_perm_dict[file_name] = np.zeros(len(path_dict[file_name]))
for j, x in enumerate(path_dict[file_name][:-1]):
sub_perm_dict[file_name][x] = j
pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y
max_f1 = 0
for thresh in np.arange(0.2, 0.601, 0.01):
thresh = np.round(thresh, 2)
 current_f1 = metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))
print("ค่า F1 ที่ threshold %.2f is %.4f" %(thresh, current_f1))
if current_f1 > max_f1:
best_thresh = thresh
max_f1 = current_f1
print('ค่า threshold ที่ดีที่สุดคือ %.2f ให้ค่า F1 เท่ากับ %.4f' %(best_thresh,max_f1))<save_to_csv>
def get_dist(x1, y1, x2, y2):
return np.sqrt(( x1 - x2)** 2 +(y1 - y2)** 2 )
pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<define_variables>
city_X = np.zeros(cities_numb)
 city_Y = np.zeros(cities_numb)
 for city, x, y in zip(cities.CityId.tolist() , cities.X.tolist() , cities.Y.tolist()):
city_X[city] = x
city_Y[city] = y
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
def lwlrap(scores, truth, **kwargs):
score, weight = calculate_per_class_lwlrap(to_np(truth), to_np(scores))
return torch.Tensor([(score * weight ).sum() ] )<load_pretrained>
def save_in_file_fast(arr, file_name):
pickle.dump(arr, open(file_name, 'wb'))
def load_from_file_fast(file_name):
return pickle.load(open(file_name, 'rb'))
def load_pkl(filename):

 with open(filename, 'rb')as f:
return pickle.load(f )<load_from_csv>
def SieveOfEratosthenes(n):
prime = [True for i in range(n + 1)]
prime[0] = False
prime[1] = False
p = 2
while(p * p <= n):
if(prime[p] == True):
for i in range(p * 2, n + 1, p):
prime[i] = False
p += 1
return prime
testing_df_X = pd.read_csv('.. /input/freesound-audio-tagging-2019/sample_submission.csv')
 testing_df_LH = pd.read_csv('.. /input/freesound-audio-tagging-2019/sample_submission.csv')
 testing_df = pd.read_csv('.. /input/freesound-audio-tagging-2019/sample_submission.csv' )<import_modules>
def get_primes() :
cache_path = 'prime_list.pkl'
if not os.path.isfile(cache_path):
n = 200000
prime = SieveOfEratosthenes(n)
 plist = []
for p in range(2, n):
if prime[p]:
plist.append(p)
 save_in_file_fast(set(plist), cache_path)
 else:
plist = load_from_file_fast(cache_path)
 return plist
class EasyDict(dict):
def __init__(self, d=None, **kwargs):
if d is None:
d = {}
if kwargs:
d.update(**kwargs)
 for k, v in d.items() :
setattr(self, k, v)
 for k in self.__class__.__dict__.keys() :
if not(k.startswith('__')and k.endswith('__')) and not k in('update', 'pop'):
setattr(self, k, getattr(self, k))
def __setattr__(self, name, value):
if isinstance(value,(list, tuple)) :
value = [self.__class__(x)
 if isinstance(x, dict)else x for x in value]
elif isinstance(value, dict)and not isinstance(value, self.__class__):
value = self.__class__(value)
 super(EasyDict, self ).__setattr__(name, value)
 super(EasyDict, self ).__setitem__(name, value)
 __setitem__ = __setattr__
def update(self, e=None, **f):
d = e or dict()
d.update(f)
 for k in d:
setattr(self, k, d[k])
 def pop(self, k, d=None):
delattr(self, k)
 return super(EasyDict, self ).pop(k, d )<load_pretrained>
plist = get_primes()
def read_audio(conf, pathname, trim_long_data):
y, sr = librosa.load(pathname, sr=conf.sampling_rate)
 if 0 < len(y):
y, _ = librosa.effects.trim(y)
 if len(y)> conf.samples:
if trim_long_data:
y = y[0:0+conf.samples]
else:
padding = conf.samples - len(y)
 offset = padding // 2
y = np.pad(y,(offset, conf.samples - len(y)- offset), 'constant')
 return y
def audio_to_melspectrogram(conf, audio):
spectrogram = librosa.feature.melspectrogram(audio,
sr=conf.sampling_rate,
n_mels=conf.n_mels,
hop_length=conf.hop_length,
n_fft=conf.n_fft,
fmin=conf.fmin,
fmax=conf.fmax)
 spectrogram = librosa.power_to_db(spectrogram)
 spectrogram = spectrogram.astype(np.float32)
 return spectrogram
def melspectrogram_to_delta(mels):
return librosa.feature.delta(mels)
 def show_melspectrogram(conf, mels, title='Log-frequency power spectrogram'):
librosa.display.specshow(mels, x_axis='time', y_axis='mel',
sr=conf.sampling_rate, hop_length=conf.hop_length,
fmin=conf.fmin, fmax=conf.fmax)
 plt.colorbar(format='%+2.0f dB')
 plt.title(title)
 plt.show()
def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):
x = read_audio(conf, pathname, trim_long_data)
 mels = audio_to_melspectrogram(conf, x)
 if debug_display:
delta = melspectrogram_to_delta(mels)
 delta_squared = melspectrogram_to_delta(delta)
 IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))
show_melspectrogram(conf, mels)
 show_melspectrogram(conf, delta)
 show_melspectrogram(conf, delta_squared)
 return mels

 def mels_preprocessing(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):
X = np.stack([X, X, X], axis=-1)
 mean = mean or X.mean()
std = std or X.std()
Xstd =(X - mean)/(std + eps)
 _min, _max = Xstd.min() , Xstd.max()
norm_max = norm_max or _max
norm_min = norm_min or _min
if(_max - _min)> eps:
V = Xstd
V[V < norm_min] = norm_min
V[V > norm_max] = norm_max
V = 255 *(V - norm_min)/(norm_max - norm_min)
 V = V.astype(np.uint8)
 else:
V = np.zeros_like(Xstd, dtype=np.uint8)
 return V
def convert_wav_to_image(df, source, img_dest, conf):
X = []
for i, row in tqdm_notebook(df.iterrows()):
x1 = read_as_melspectrogram(conf, source/str(row.fname), trim_long_data=False)
 x1 = mels_preprocessing(x1)
 X.append(x1)
 return df, X
def split_long_data(conf, X):
L = X.shape[1]
one_length = conf['dims'][1]
loop_length = int(one_length * 0.9)
 min_length = int(one_length * 0.2)
 for idx in range(L // loop_length):
cur = loop_length * idx
rest = L - cur
if one_length <= rest:
yield X[:, cur:cur+one_length]
elif min_length <= rest:
cur = L - one_length
yield X[:, cur:cur+one_length]
def convert_X(df, conf, datapath):
df_x = pd.DataFrame({'fname': [], 'labels': []})
 X = []
for i, row in tqdm_notebook(df.iterrows()):
j = 1
x1 = read_as_melspectrogram(conf, datapath / row.fname, trim_long_data=False)
 x1 = mels_preprocessing(x1)
 for chunk in split_long_data(conf, x1):
X.append(np.expand_dims(chunk, axis=-1))
df_x = df_x.append({'fname': str(j)+ '-' + df.fname[i], 'labels':df.labels[i]}, ignore_index=True)
 f = open('/kaggle/working/trn_curated_X/' + str(j)+ '-' + df.fname[i],"w+")
 j += 1
return df_x, X
def convert_X_test(df, conf, datapath):
df_x = pd.DataFrame({'fname': []})
 X = []
for i, row in tqdm_notebook(df.iterrows()):
j = 1
x1 = read_as_melspectrogram(conf, datapath / row.fname, trim_long_data=False)
 x1 = mels_preprocessing(x1)
 for chunk in split_long_data(conf, x1):
X.append(np.expand_dims(chunk, axis=-1))
df_x = df_x.append({'fname': str(j)+ '-' + df.fname[i]}, ignore_index=True)
 f = open('/kaggle/working/test_X/' + str(j)+ '-' + df.fname[i],"w+")
 j += 1
return df_x, X<init_hyperparams>
def get_score(santa_path, numb_list):
sum_dist = 0
for i in range(len(santa_path)- 1):
city1 = santa_path[i]
city2 = santa_path[i+1]
if city1 > city2:
city1, city2 = city2, city1
x1 = city_X[city1]
y1 = city_Y[city1]
x2 = city_X[city2]
y2 = city_Y[city2]
city1_dict = dist_dict[city1]
if city2 in city1_dict:
dist = city1_dict[city2]
else:
dist = get_dist(x1, y1, x2, y2)
 dist_dict[city1][city2] = dist
if(( numb_list[i] + 1)% 10 == 0)and(not santa_path[i] in plist):
dist *= 1.1
sum_dist += dist
return sum_dist
conf_LH = EasyDict()
conf_LH.sampling_rate = 44100
conf_LH.duration = 2
conf_LH.hop_length = 347 * conf_LH.duration
conf_LH.fmin = 20
conf_LH.fmax = conf_LH.sampling_rate // 2
conf_LH.n_mels = 128
conf_LH.n_fft = conf_LH.n_mels * 20
conf_LH.samples = conf_LH.sampling_rate * conf_LH.duration
conf_LH.dims =(conf_LH.n_mels, 1 + int(np.floor(conf_LH.samples/conf_LH.hop_length)) , 1)
 conf_X = EasyDict()
conf_X.sampling_rate = 44100
conf_X.duration = 8
conf_X.hop_length = 347 * conf_X.duration
conf_X.fmin = 20
conf_X.fmax = conf_X.sampling_rate // 2
conf_X.n_mels = 128
conf_X.n_fft = conf_X.n_mels * 20
conf_X.samples = conf_X.sampling_rate * conf_X.duration
conf_X.dims =(conf_X.n_mels, 1 + int(np.floor(conf_X.samples/conf_X.hop_length)) , 1 )<categorify>
best_path = subm['Path'].tolist()
testing_df_X, X_test_X = convert_wav_to_image(testing_df, source=Path('.. /input/freesound-audio-tagging-2019/test'), img_dest=Path('test'), conf = conf_X)
 testing_df_LH, X_test_LH = convert_wav_to_image(testing_df, source=Path('.. /input/freesound-audio-tagging-2019/test'), img_dest=Path('test'), conf = conf_LH)
 print(f"Finished data conversion at {(time.time() -start_time)/3600} hours")
 print(len(X_test_X))
print(len(testing_df_X))
print(len(X_test_LH))
print(len(testing_df_LH))<load_from_csv>
dist_dict = {i:{} for i in range(cities_numb)}
CUR_X_FILES, CUR_X = list(testing_df_X.fname.values), X_test_X
test_X = ImageList.from_csv(Path('/kaggle/working'), Path('.. /input/freesound-audio-tagging-2019/sample_submission.csv'), folder=Path('.. /input/freesound-audio-tagging-2019/test'))
predictions_X = torch.from_numpy(np.zeros(( 3361,80)) ).float()
print(predictions_X.shape)
 num_folds = 5
for i in range(num_folds):
learn_X = load_learner(path = '.', file=Path('/kaggle/input/fat-curated-ensemble-mixup-tta/FAT2019_X_1D_mixup_TTA_Kfold_'+str(i)+'.pkl'), test=test_X)
 preds_X, _ = learn_X.TTA(ds_type=DatasetType.Test)
 print(preds_X.shape)
 predictions_X = predictions_X + preds_X
predictions_X = predictions_X/num_folds
testing_df_X[learn_X.data.classes] = predictions_X
testing_df_X.head()<load_from_csv>
def pop(cur_perm, numb_list, score):
city1 = cur_perm[0]
city2 = cur_perm[1]
x1 = city_X[city1]
y1 = city_Y[city1]
x2 = city_X[city2]
y2 = city_Y[city2]
dist = get_dist(x1, y1, x2, y2)
 pos = numb_list[0]
if(( pos + 1)% 10 == 0 and(not city1 in plist)) :
return cur_perm[1:], numb_list[1:], score - dist * 1.1
else:
return cur_perm[1:], numb_list[1:], score - dist
def push(cur_perm, numb_list, score, x):
city1 = cur_perm[-1]
city2 = x
x1 = city_X[city1]
y1 = city_Y[city1]
x2 = city_X[city2]
y2 = city_Y[city2]
dist = get_dist(x1, y1, x2, y2)
 pos = numb_list[-1] + 1
if(( pos)% 10 == 0 and(not city1 in plist)) :
return cur_perm + [x], numb_list + [pos], score + dist * 1.1
else:
return cur_perm + [x], numb_list + [pos], score + dist
<load_from_csv>
def improve_perm(santa_path, len_perm):
santa_len = len(santa_path)
 L = 150000
R = len_perm + L
numb_list = list(range(L, R))
cur_perm = santa_path[L:R]
cur_dist = get_score(cur_perm, numb_list)
 for i in tqdm(range(0, santa_len - R - 1)) :
if i % 15 == 0:
for file_name in filenames:
new_path = []
indx = sorted([sub_perm_dict[file_name][x] for x in cur_perm[1:-1]])
 for ind in indx:
new_path += [path_dict[file_name][int(ind)]]
cur_path = [cur_perm[0]] + new_path + [cur_perm[-1]]
cur_path_rev = [cur_perm[0]] + new_path[::-1] + [cur_perm[-1]]
cur_score = get_score(cur_path, numb_list)
 if(cur_score < cur_dist and cur_dist - cur_score > 0.05):
print('GO ', cur_score, cur_dist, get_score(cur_perm, numb_list))
ans = list(santa_path)
 ans[L:R] = cur_path
return ans
cur_score = get_score(cur_path_rev, numb_list)
 if(cur_score < cur_dist and cur_dist - cur_score > 0.05):
print('REV ', cur_score, cur_dist)
 ans = list(santa_path)
 ans[L:R] = cur_path_rev
return ans
cur_perm, numb_list, cur_dist = pop(cur_perm, numb_list, cur_dist)
 cur_perm, numb_list, cur_dist = push(cur_perm, numb_list, cur_dist, santa_path[R])
 L += 1
R += 1
CUR_X_FILES, CUR_X = list(testing_df_LH.fname.values), X_test_LH
test_LH = ImageList.from_csv(Path('/kaggle/working'), Path('.. /input/freesound-audio-tagging-2019/sample_submission.csv'), folder=Path('.. /input/freesound-audio-tagging-2019/test'))
predictions_LH = torch.from_numpy(np.zeros(( 3361,80)) ).float()
print(predictions_LH.shape)
 num_folds = 5
for i in range(num_folds):
learn_LH = load_learner(path = '.', file=Path('/kaggle/input/fat-curated-ensemble-mixup-tta/FAT2019_LH_1D_mixup_TTA_Kfold_'+str(i)+'.pkl'), test=test_LH)
 preds_LH, _ = learn_LH.TTA(ds_type=DatasetType.Test)
 print(preds_LH.shape)
 predictions_LH = predictions_LH + preds_LH
predictions_LH = predictions_LH/num_folds
testing_df_LH[learn_LH.data.classes] = predictions_LH
testing_df_LH.head()<load_from_csv>
best_path2 = improve_perm(best_path, 100 )
<save_to_csv>
get_score(best_path, list(range(len(best_path))))
preds =(predictions_X + predictions_LH)/ 2
testing_df[learn_X.data.classes] = preds
testing_df.to_csv('submission.csv', index=False)
 testing_df.head(20 )<import_modules>
get_score(best_path2, list(range(len(best_path2))))
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms<set_options>
ans = pd.DataFrame()
ans['Path'] = best_path2
<set_options><EOS> 
ans.to_csv('submission.csv', index = None )