Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<define_variables>
init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )
initial_path = check_reverse(path.copy() )<train_model>
train_data = pd.read_json('.. /input/train.json')
 test_data = pd.read_json('.. /input/test.json' )
new_path = optN(initial_path, tuples[0], 11, 0 )<compute_test_metric>
print("The training data consists of {} recipes".format(len(train_data)) )
print(f'Total score after stage 1 is {score_path(new_path):.2f} in {(time.time() -t1)/60.0:.1f} min' )<train_model>
print("Number of cuisine categories: {}".format(len(train_data.cuisine.unique())))
 train_data.cuisine.unique()
t2 = time.time()
new_path1 = optN(new_path.copy() , tuples[1], 11, 0 )<compute_test_metric>
def random_colours(number_of_colors):

 colors = []
for i in range(number_of_colors):
colors.append("
return colors
print(f'Total score after stage 2 is {score_path(new_path1):.2f} in {(time.time() -t2)/60.0:.1f} min' )<prepare_output>
labelpercents = []
for i in train_data.cuisine.value_counts() :
percent =(i/sum(train_data.cuisine.value_counts())) *100
percent = "%.2f" % percent
percent = str(percent + '%')
 labelpercents.append(percent )
t3 = time.time()
new_path2 = optN(new_path1.copy() , tuples[2], 11, 0 )<compute_test_metric>
longrecipes = train_data[train_data['ingredients'].str.len() > 30]
print("It seems that {} recipes consist of more than 30 ingredients!".format(len(longrecipes)) )
print(f'Total score after stage 3 is {score_path(new_path2):.2f} in {(time.time() -t3)/60.0:.1f} min' )<train_model>
shortrecipes = train_data[train_data['ingredients'].str.len() <= 2]
print("It seems that {} recipes consist of less than or equal to 2 ingredients!".format(len(shortrecipes)) )
print(f'Total local optimizations time is {(time.time() -t1)/60.0:.1f} min' )<compute_test_metric>
print("Explore the ingredients in the shortest recipes in our training set:" + "
")
 print(list(train_data[train_data['ingredients'].str.len() == 1].ingredients.values))
print("And there corresponding labels" + "
")
 print(list(train_data[train_data['ingredients'].str.len() == 1].cuisine.values))
print(f'Final score is {score_path(new_path2):.2f}' )<save_to_csv>
allingredients = []
for item in train_data['ingredients']:
for ingr in item:
allingredients.append(ingr )
pd.DataFrame({'Path': new_path2} ).to_csv('submission.csv', index=False )<train_model>
countingr = Counter()
for ingr in allingredients:
countingr[ingr] += 1
print(f'Total kernel time is {(time.time() -t0)/60.0:.1f} min' )<train_model>
print("The most commonly used ingredients(with counts)are:")
 print("
")
 print(countingr.most_common(20))
print("
")
 print("The number of unique ingredients in our training sample is {}.".format(len(countingr)) )
<import_modules>
features = []
for item in train_data['ingredients']:
features.append(item )
import ast
import numba
import string
import numpy as np
import pandas as pd
from itertools import combinations, permutations, product
from math import sqrt, factorial
from sklearn.neighbors import KDTree
from sympy import isprime, primerange
from tqdm import tqdm_notebook<define_variables>
ingredients = []
for item in train_data['ingredients']:
for ingr in item:
ingredients.append(ingr )
K = 4<statistical_test>
tfidf = TfidfVectorizer(vocabulary= list(set([str(i ).lower() for i in ingredients])) , max_df=0.99, norm='l2', ngram_range=(1, 4))
X_tr = tfidf.fit_transform([str(i)for i in features])
 feature_names = tfidf.get_feature_names()
def frunkopt_func_generator(K):
K = K - 1
letters = np.array(list(string.ascii_uppercase[:K]))
i = 0
tab = ' '
function_name = f'move_{K}opt'
function_strings = []
function_strings.append("@numba.jit('void(i8[:], i8[:], i8)', nopython=True, parallel=False)")
 function_strings.append(f"def {function_name}(path, idx, move_type):")
 function_strings.append("{}pslice = slice(idx[0]+1, idx[-1]+1)".format(tab))
function_strings.append("{}{} = {}".format(tab, ', '.join(letters), ', '.join([f'path[idx[{t}]+1:idx[{t+1}]+1]' for t in np.arange(K)])))
 function_strings.append("{}{} = {}".format(tab, ', '.join(string.ascii_lowercase[:K]), '[::-1], '.join(letters)+ '[::-1]'))
for p in permutations(np.arange(K)) :
for r in product([0, 1], repeat=K):
out_arr = [letters[j].lower() if r[j] else letters[j] for j in p]
function_strings.append('{}{} move_type == {}:'.format(tab, 'elif' if i else 'if', i))
function_strings.append("{}path[pslice] = np.concatenate(( {})) ".format(tab * 2, ', '.join(out_arr)))
 i += 1
return function_name,.format('
'.join(function_strings))<feature_engineering>
def top_feats_by_class(trainsample,target,featurenames, min_tfidf=0.1, top_n=10):

 dfs = []
labels = np.unique(target)
 for label in labels:
ids = np.where(target==label)
 D = trainsample[ids].toarray()
D[D < min_tfidf] = 0
tfidf_means = np.nanmean(D, axis=0)
 topn_ids = np.argsort(tfidf_means)[::-1][:top_n]
top_feats = [(featurenames[i], tfidf_means[i])for i in topn_ids]
df = pd.DataFrame(top_feats)
 df.columns = ['feature', 'tfidf']
df['cuisine'] = label
dfs.append(df)
 return dfs
frunktion_name, frunktion_body = frunkopt_func_generator(K )<find_best_params>
target = train_data['cuisine']
frunkopt_move = locals() [frunktion_name]<load_from_csv>
result_tfidf = top_feats_by_class(X_tr, target, feature_names, min_tfidf=0.1, top_n=5 )
cities = pd.read_csv(
 '.. /input/traveling-santa-2018-prime-paths/cities.csv',
index_col=['CityId'])
 XY = np.stack(
(
 cities.X.astype(np.float32),
cities.Y.astype(np.float32)
),
axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<compute_train_metric>
print("How training data looks like at this stage(example of one recipe):")
 print(str(features[0])+ '
')
 print("Number of instances: "+ str(len(features)) + '
')
 print("And the target variable:")
 print(target[0] )
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_dist(id_from, id_to, offset):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i8[:], i8)', nopython=True, parallel=False)
 def chunk_score(chunk, offset):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def path_score(path):
return chunk_score(path, 0)
 def path_score_full(path):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9
for path_index in numba.prange(path.shape[0] - 1):
id_from, id_to = path[path_index], path[path_index + 1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return(
 round(pure_distance, 4),
round(0.1 * penalty, 4),
round(pure_distance + 0.1 * penalty, 4)
 )<define_variables>
features_test = []
for item in test_data['ingredients']:
features_test.append(item )
N_NEIGHBORS = 2*K + 2
RADIUS = 3*K + 3<find_best_params>
print("How test data looks like at this stage(example of one recipe):")
 print(str(features_test[0])+ '
')
 print("Number of instances: "+ str(len(features_test)) )
kdt = KDTree(XY[1:])
 neighbors_N = kdt.query(XY[1:], N_NEIGHBORS, return_distance=False)[:, K:]
neighbors_R = kdt.query_radius(XY[1:], RADIUS, count_only=False, return_distance=False )<count_unique_values>
features_processed= []
for item in features:
newitem = []
for ingr in item:
ingr.lower()
ingr = re.sub("[^a-zA-Z]"," ",ingr)
 ingr = re.sub(( r'\b(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\b'), ' ', ingr)
 newitem.append(ingr)
 features_processed.append(newitem)
 features_test_processed= []
for item in features_test:
newitem = []
for ingr in item:
ingr.lower()
ingr = re.sub("[^a-zA-Z]"," ",ingr)
 ingr = re.sub(( r'\b(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\b'), ' ', ingr)
 newitem.append(ingr)
 features_test_processed.append(newitem )
neighbors = set()
for city_id in tqdm_notebook(cities.index[1:]):
for neib_triplet in combinations(neighbors_N[city_id - 1] + 1, K):
neighbors.add(tuple(sorted(neib_triplet)))
 for neib_triplet in combinations(neighbors_R[city_id - 1][:N_NEIGHBORS-K] + 1, K):
neighbors.add(tuple(sorted(neib_triplet)))
 print(f'{len(neighbors)} cities {K}-neighbors are selected.' )<statistical_test>
count_m = []
for recipe in features_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed training sample: " + str(len(count_m)) )
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def sum_distance(ids):
res = 0
for i in numba.prange(len(ids)) :
for j in numba.prange(i + 1, len(ids)) :
res += cities_dist(ids[i], ids[j], 0)
 return res<load_from_csv>
count_m = []
for recipe in features_test_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed test sample: " + str(len(count_m)) )
initial_path = pd.read_csv('.. /input/not-a-5-and-5-halves-opt-0efc12/1515559.6779840707.csv' ).Path.values<compute_test_metric>
vectorizer = CountVectorizer(analyzer = "word",
ngram_range =(1,1),
binary = True,
tokenizer = None,
preprocessor = None,
stop_words = None,
max_df = 0.99 )
path = initial_path.copy()
path_index = np.argsort(path[:-1])
 initial_score = total_score = path_score(path)
 print(path_score_full(path))
print(f'Total score is {path_score(path):.2f}.' )<define_variables>
train_X = vectorizer.fit_transform([str(i)for i in features_processed])
 test_X = vectorizer.transform([str(i)for i in features_test_processed] )
frunkopt_moves_count = factorial(K - 1)* 2 **(K - 1)
 print(frunkopt_moves_count )<define_variables>
lb = LabelEncoder()
train_Y = lb.fit_transform(target )
runs = 2
neighbors_len = len(neighbors)
 cases_improved = np.zeros(( runs, frunkopt_moves_count))<save_to_csv>
vclf=VotingClassifier(estimators=[('clf1',LogisticRegression(random_state = 42)) , 
('clf2',SVC(kernel='linear',random_state = 42,probability=True)) , 
('clf3',RandomForestClassifier(n_estimators = 600,random_state = 42)) ],
voting='soft', weights = [1,1,1])
 vclf.fit(train_X, train_Y )
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )<find_best_params>
kfold = model_selection.KFold(n_splits=10, random_state=42)
 valscores = model_selection.cross_val_score(vclf, train_X, train_Y, cv=kfold)
 print('Mean accuracy on 10-fold cross validation: ' + str(np.mean(valscores)) )
<compute_test_metric><EOS> 
predictions = vclf.predict(test_X)
 predictions = lb.inverse_transform(predictions)
 predictions_final = pd.DataFrame({'cuisine' : predictions , 'id' : test_data.id }, columns=['id', 'cuisine'])
 predictions_final.to_csv('Final_submission.csv', index = False )