Kaggle competition: Traveling Santa 2018 - Prime Paths
<SOS> metric: TravelingSanta2 Kaggle data source: traveling-santa-2018-prime-paths<choose_model_class>
import ast
import numba
import string
import numpy as np
import pandas as pd
from itertools import combinations, permutations, product
from math import sqrt, factorial
from sklearn.neighbors import KDTree
from sympy import isprime, primerange
from tqdm import tqdm_notebook
CUR_X_FILES, CUR_X = list(df.fname.values), X_train
learn = cnn_learner(data, models.resnet18, pretrained=False, metrics=[lwlrap])
 learn.load('fat2019_fastai_cnn2d_stage-2');<import_modules>
K = 4
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms<set_options>
def frunkopt_func_generator(K):
K = K - 1
letters = np.array(list(string.ascii_uppercase[:K]))
i = 0
tab = ' '
function_name = f'move_{K+1}opt'
function_strings = []
function_strings.append("@numba.jit('void(i8[:], i8[:], i8)', nopython=True, parallel=False)")
 function_strings.append(f"def {function_name}(path, idx, move_type):")
 function_strings.append("{}pslice = slice(idx[0]+1, idx[-1]+1)".format(tab))
function_strings.append("{}{} = {}".format(tab, ', '.join(letters), ', '.join([f'path[idx[{t}]+1:idx[{t+1}]+1]' for t in np.arange(K)])))
 function_strings.append("{}{} = {}".format(tab, ', '.join(string.ascii_lowercase[:K]), '[::-1], '.join(letters)+ '[::-1]'))
for p in permutations(np.arange(K)) :
for r in product([0, 1], repeat=K):
out_arr = [letters[j].lower() if r[j] else letters[j] for j in p]
function_strings.append('{}{} move_type == {}:'.format(tab, 'elif' if i else 'if', i))
function_strings.append("{}path[pslice] = np.concatenate(( {})) ".format(tab * 2, ', '.join(out_arr)))
 i += 1
return function_name,.format('
'.join(function_strings))
torch.cuda.is_available()<set_options>
frunktion_name, frunktion_body = frunkopt_func_generator(K )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 520
seed_everything(SEED )<feature_engineering>
frunkopt_move = locals() [frunktion_name]
N_JOBS = cpu_count()
os.environ['MKL_NUM_THREADS'] = str(N_JOBS)
 os.environ['OMP_NUM_THREADS'] = str(N_JOBS)
 DataLoader = partial(DataLoader, num_workers=N_JOBS )<define_variables>
cities = pd.read_csv(
 '.. /input/traveling-santa-2018-prime-paths/cities.csv',
index_col=['CityId'])
 XY = np.stack(
(
 cities.X.astype(np.float32),
cities.Y.astype(np.float32)
),
axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_dist(id_from, id_to, offset):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i8[:], i8)', nopython=True, parallel=False)
 def chunk_score(chunk, offset):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def path_score(path):
return chunk_score(path, 0)
 def path_score_full(path):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9
for path_index in numba.prange(path.shape[0] - 1):
id_from, id_to = path[path_index], path[path_index + 1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return(
 round(pure_distance, 4),
round(0.1 * penalty, 4),
round(pure_distance + 0.1 * penalty, 4)
 )
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )<define_variables>
N_NEIGHBORS = 2*K + 2
RADIUS = 3*K + 3
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}<load_from_csv>
kdt = KDTree(XY[1:])
 neighbors_N = kdt.query(XY[1:], N_NEIGHBORS, return_distance=False)[:, :2*K-1]
neighbors_R = kdt.query_radius(XY[1:], RADIUS, count_only=False, return_distance=False )
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()<load_from_csv>
neighbors = set()
for city_id in tqdm_notebook(cities.index[1:]):
for neib_triplet in combinations(neighbors_N[city_id - 1] + 1, K):
neighbors.add(tuple(sorted(neib_triplet)))
 for neib_triplet in combinations(neighbors_R[city_id - 1][:N_NEIGHBORS-K+1] + 1, K):
neighbors.add(tuple(sorted(neib_triplet)))
 print(f'{len(neighbors)} cities {K}-neighbors are selected.' )
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()<data_type_conversions>
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def sum_distance(ids):
res = 0
for i in numba.prange(len(ids)) :
for j in numba.prange(i + 1, len(ids)) :
res += cities_dist(ids[i], ids[j], 0)
 return res
labels = test_df.columns[1:].tolist()
labels<define_variables>
initial_path = pd.read_csv('.. /input/dp-shuffle/submission.csv' ).Path.values
num_classes = len(labels)
 num_classes<prepare_x_and_y>
path = initial_path.copy()
path_index = np.argsort(path[:-1])
 initial_score = total_score = path_score(path)
 print(path_score_full(path))
print(f'Total score is {path_score(path):.2f}.' )
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape<load_pretrained>
frunkopt_moves_count = factorial(K - 1)* 2 **(K - 1)
 print(frunkopt_moves_count )
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )<categorify>
runs = 2
neighbors_len = len(neighbors)
 cases_improved = np.zeros(( runs, frunkopt_moves_count))
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - base_dim)
 image = image.crop([crop, 0, crop + base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname<categorify>
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )
transforms_dict = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}<define_search_model>
for run in np.arange(1, runs + 1):
magic_number = K *(10 **(RADIUS - N_NEIGHBORS + 1)) // run
print(f'Run
for step, ids in tqdm_notebook(enumerate(neighbors[:magic_number], 1), total=magic_number):
if step % 10 ** 6 == 0:
new_total_score = path_score(path)
 last_improvement = total_score - new_total_score
print(f"score {new_total_score:.2f} | last 10^6 {last_improvement:.2f} | total {initial_score - new_total_score:.2f}.")
 print(cases_improved[run - 1])
 total_score = new_total_score
if last_improvement > K:
make_submission(f'frunkopt_{path_score(path):.4f}', path)
 idx = sorted(path_index[ids])
 new_idx = idx - idx[0]
pslice = slice(idx[0], idx[-1] + 2)
 chunk = path[pslice]
best_score = chunk_score(chunk, idx[0])
 best_move = -1
for move_type in numba.prange(1, frunkopt_moves_count):
new_chunk = chunk.copy()
frunkopt_move(new_chunk, new_idx, move_type)
 new_score = chunk_score(new_chunk, idx[0])
 if new_score < best_score:
best_score = new_score
best_move = move_type
best_chunk = new_chunk.copy()
if best_move > -1:
path[pslice] = best_chunk
path_index = np.argsort(path[:-1])
 cases_improved[run - 1, best_move] += 1
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x<choose_model_class>
total_score = path_score(path)
 print(f'Total improvement | {initial_score - total_score:.2f}')
 print('Final scores |', path_score_full(path))
<choose_model_class><EOS> 
make_submission(f'final_frunkopt_{path_score(path):.4f}', path )