Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<find_best_params>
tqdm.pandas()
frunkopt_move = locals() [frunktion_name]<load_from_csv>
train = pd.read_json('.. /input/train.json')
 test = pd.read_json('.. /input/test.json' )
cities = pd.read_csv(
 '.. /input/traveling-santa-2018-prime-paths/cities.csv',
index_col=['CityId'])
 XY = np.stack(
(
 cities.X.astype(np.float32),
cities.Y.astype(np.float32)
),
axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<compute_train_metric>
train['num_ingredients'] = train['ingredients'].apply(len)
 train = train[train['num_ingredients'] > 1]
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_dist(id_from, id_to, offset):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i8[:], i8)', nopython=True, parallel=False)
 def chunk_score(chunk, offset):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def path_score(path):
return chunk_score(path, 0)
 def path_score_full(path):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9
for path_index in numba.prange(path.shape[0] - 1):
id_from, id_to = path[path_index], path[path_index + 1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return(
 round(pure_distance, 4),
round(0.1 * penalty, 4),
round(pure_distance + 0.1 * penalty, 4)
 )<define_variables>
lemmatizer = WordNetLemmatizer()
def preprocess(ingredients):
ingredients_text = ' '.join(ingredients)
 ingredients_text = ingredients_text.lower()
ingredients_text = ingredients_text.replace('-', ' ')
 words = []
for word in ingredients_text.split() :
if re.findall('[0-9]', word): continue
if len(word)<= 2: continue
if '’' in word: continue
word = lemmatizer.lemmatize(word)
 if len(word)> 0: words.append(word)
 return ' '.join(words)
 for ingredient, expected in [ 
('Eggs', 'egg'), 
('all-purpose flour', 'all purpose flour'), 
('purée', 'purée'), 
('1% low-fat milk', 'low fat milk'), 
('half & half', 'half half'), 
('safetida(powder)', 'safetida(powder)')
 ]:
actual = preprocess([ingredient])
 assert actual == expected, f'"{expected}" is excpected but got "{actual}"'
N_NEIGHBORS = 2*K + 2
RADIUS = 3*K + 3<find_best_params>
train['x'] = train['ingredients'].progress_apply(preprocess)
 test['x'] = test['ingredients'].progress_apply(preprocess)
 train.head()
kdt = KDTree(XY[1:])
 neighbors_N = kdt.query(XY[1:], N_NEIGHBORS, return_distance=False)[:, :2*K-1]
neighbors_R = kdt.query_radius(XY[1:], RADIUS, count_only=False, return_distance=False )<count_unique_values>
vectorizer = make_pipeline(
 TfidfVectorizer(sublinear_tf=True),
FunctionTransformer(lambda x: x.astype('float16'), validate=False)
)
 x_train = vectorizer.fit_transform(train['x'].values)
 x_train.sort_indices()
x_test = vectorizer.transform(test['x'].values )
neighbors = set()
for city_id in tqdm_notebook(cities.index[1:]):
for neib_triplet in combinations(neighbors_N[city_id - 1] + 1, K):
neighbors.add(tuple(sorted(neib_triplet)))
 for neib_triplet in combinations(neighbors_R[city_id - 1][:N_NEIGHBORS-K+1] + 1, K):
neighbors.add(tuple(sorted(neib_triplet)))
 print(f'{len(neighbors)} cities {K}-neighbors are selected.' )<statistical_test>
label_encoder = LabelEncoder()
y_train = label_encoder.fit_transform(train['cuisine'].values)
 dict(zip(label_encoder.classes_, label_encoder.transform(label_encoder.classes_)) )
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def sum_distance(ids):
res = 0
for i in numba.prange(len(ids)) :
for j in numba.prange(i + 1, len(ids)) :
res += cities_dist(ids[i], ids[j], 0)
 return res<load_from_csv>
estimator = SVC(
 C=80,
kernel='rbf',
gamma=1.7,
coef0=1,
cache_size=500, 
)
 classifier = OneVsRestClassifier(estimator, n_jobs=-1 )
initial_path = pd.read_csv('.. /input/dp-shuffle/submission.csv' ).Path.values<compute_test_metric>
%%time
scores = cross_validate(classifier, x_train, y_train, cv=3)
 scores['test_score'].mean()
path = initial_path.copy()
path_index = np.argsort(path[:-1])
 initial_score = total_score = path_score(path)
 print(path_score_full(path))
print(f'Total score is {path_score(path):.2f}.' )<define_variables>
%%time
classifier.fit(x_train, y_train )
frunkopt_moves_count = factorial(K - 1)* 2 **(K - 1)
 print(frunkopt_moves_count )<define_variables>
y_pred = label_encoder.inverse_transform(classifier.predict(x_train))
y_true = label_encoder.inverse_transform(y_train)
 print(f'accuracy score on train data: {accuracy_score(y_true, y_pred)}')
 def report2dict(cr):
rows = []
for row in cr.split("
"):
parsed_row = [x for x in row.split(" ")if len(x)> 0]
if len(parsed_row)> 0: rows.append(parsed_row)
 measures = rows[0]
classes = defaultdict(dict)
 for row in rows[1:]:
class_label = row[0]
for j, m in enumerate(measures):
classes[class_label][m.strip() ] = float(row[j + 1].strip())
 return classes
report = classification_report(y_true, y_pred)
 pd.DataFrame(report2dict(report)).T
<save_to_csv><EOS> 
y_pred = label_encoder.inverse_transform(classifier.predict(x_test))
test['cuisine'] = y_pred
test[['id', 'cuisine']].to_csv('submission.csv', index=False)
 test[['id', 'cuisine']].head()