Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<load_from_csv>
import json
import numpy as np
path_init = pd.read_csv('.. /input/lkh-solver/submission.csv')['Path'].tolist()
score_init = compute_score(path_init)
 score_init<prepare_x_and_y>
with open('.. /input/train.json')as file:
training_data = json.load(file)
 with open('.. /input/test.json')as file:
testing_data = json.load(file )
path = path_init
path = path[ : len(path)-1]
cities_path = cities.reindex(path)
 cityId = cities_path.CityId.values
isPrime = cities_path.IsPrime.values
coord = cities_path[['X', 'Y']].values
cities_path.tail()<define_variables>
def cleanup_ingredient(ingredient):
x = re.sub('('s)|(\’s)','',ingredient)
 x = re.sub('[0-9%\(\)\/\.™\,®'\&]|(lb\.) |(oz\.) ','',x)
 x = re.sub('\-',' ', x)
 x = re.sub(' +',' ', x)
 x = re.sub('(low fat)|(full fat)|(reduced fat)|(fat free)|(skimmed)|(fatfree)|(lowfat)|(nonfat)|(non fat)|(low sodium)|(reduced sodium)|(less sodium)|(no salt added)|(homemade)|(gluten free)|(salt free)|(reduc sodium)|(cholesterol free)|(s real)|(light)|(free range)|(skim)','',x)
 x = re.sub('(small)|(medium)|(large)','', x)
 x = re.sub('(heinz)|(hellmannâ€)|(hellmanns)|(hellmann)|(kikkoman)|(kraft)|(taco bell)|(mccormick)|(mcintosh)|(knorr)|(or best food)|(best food)', '', x)
 x = re.sub('[éèê]', 'e', x)
 x = re.sub('[íî]', 'i', x)
 x = re.sub('[â]', 'a', x)
 x = re.sub(' +',' ', x)
 x = x.strip()
return x
for s in range(1,len(path)) :
if s%10 == 0 and not isPrime[s-1]:
t = s-1
if isPrime[t-1]:
d_10thstep_old = distance(coord[s-3], coord[s-2])+ distance(coord[s-2], coord[s-1])+ distance(coord[s-1], coord[s])* 1.1
d_10thstep_new = distance(coord[s-3], coord[s-1])+ distance(coord[s-1], coord[s-2])+ distance(coord[s-2], coord[s])
 if d_10thstep_new < d_10thstep_old :
gain = d_10thstep_new - d_10thstep_old
print(s, " gain: ", gain, " -- t = ", t)
 path[t-1] , path[s-1] = path[s-1] , path[t-1]
isPrime[t-1] , isPrime[s-1] = isPrime[s-1] , isPrime[t-1]
coord[t-1] , coord[s-1] = coord[s-1] , coord[t-1]
print("Done")
<define_variables>
from sklearn.preprocessing import LabelEncoder
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import LinearSVC, SVC
from sklearn.multiclass import OneVsRestClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
for s in range(1,len(path)) :
if s%10 == 0 and not isPrime[s-1]:
t = s+1
if isPrime[t-1]:
d_10thstep_old = distance(coord[s-2], coord[s-1])+ distance(coord[s-1], coord[s])*1.1 + distance(coord[s], coord[s+1])
 d_10thstep_new = distance(coord[s-2], coord[s])+ distance(coord[s], coord[s-1])+ distance(coord[s-1], coord[s+1])
 if d_10thstep_new < d_10thstep_old :
gain = d_10thstep_new - d_10thstep_old
print(s, " gain: ", gain, " -- t = ", t)
 path[t-1] , path[s-1] = path[s-1] , path[t-1]
isPrime[t-1] , isPrime[s-1] = isPrime[s-1] , isPrime[t-1]
coord[t-1] , coord[s-1] = coord[s-1] , coord[t-1]
print("Done")
<prepare_x_and_y>
training_recipes = [" ".join([cleanup_ingredient(ingredient)for ingredient in recipe['ingredients']])for recipe in training_data]
testing_recipes = [" ".join([cleanup_ingredient(ingredient)for ingredient in recipe['ingredients']])for recipe in testing_data]
for s in range(1,len(path)) :
if s%10 == 0 and not isPrime[s-1]:
x0 = coord[s-1][0]
y0 = coord[s-1][1]
d = distance(coord[s-2], coord[s-1])+ distance(coord[s-1], coord[s])* 1.1
cell = cities[(cities.X > x0-d)&(cities.X < x0+d)]
cell = cell[(cell.Y > y0-d)&(cell.Y < y0+d)]
cell = cell[cell.IsPrime]
coord_primes_arround = cell[['X','Y']].values
cityId_primes_arround = cell.CityId.values
for j in range(len(cell)) :
d_10thstep_old = distance(coord[s-2], coord[s-1])+ distance(coord[s-1], coord[s])* 1.1
t = path.index(cityId_primes_arround[j])+ 1
if(t%10 != 0)and(t!=s-1)and(t!=s+1):
d_prime_old = distance(coord[t-2], coord[t-1])+ distance(coord[t-1], coord[t])
 d_10thstep_new = distance(coord[s-2], coord[t-1])+ distance(coord[t-1], coord[s])
 d_prime_new = distance(coord[t-2], coord[s-1])+ distance(coord[s-1], coord[t])
 if not isprime(cityId[t-2])and(t-1)%10==0 :
d_prime_old = distance(coord[t-2], coord[t-1])*1.1 + distance(coord[t-1], coord[t])
 d_prime_new = distance(coord[t-2], coord[s-1])*1.1 + distance(coord[s-1], coord[t])
 d_old = d_10thstep_old + d_prime_old
d_new = d_10thstep_new + d_prime_new
if d_new < d_old :
gain = d_new - d_old
print(s, " gain: ", gain, " -- t = ", t)
 path[t-1] , path[s-1] = path[s-1] , path[t-1]
isPrime[t-1] , isPrime[s-1] = isPrime[s-1] , isPrime[t-1]
coord[t-1] , coord[s-1] = coord[s-1] , coord[t-1]
print("Done" )<compute_test_metric>
ingredient_encoder = TfidfVectorizer(binary=True )
path.append(0)
 score_final = compute_score(path)
 print(score_final - score_init )<save_to_csv>
X_train = ingredient_encoder.fit_transform(training_recipes )
pd.DataFrame({'Path':path} ).to_csv('submission.csv', index=False )<set_options>
X_test = ingredient_encoder.transform(testing_recipes )
pd.options.display.float_format = '{:.6f}'.format<load_from_csv>
cuisine_encoder = LabelEncoder()
y_train = cuisine_encoder.fit_transform([recipe['cuisine'] for recipe in training_data] )
def read_cities(filename='.. /input/traveling-santa-2018-prime-paths/cities.csv'):
return pd.read_csv(filename, index_col=['CityId'])
 class Tour:
cities = read_cities()
coords =(cities.X + 1j * cities.Y ).values
penalized = ~cities.index.isin(sympy.primerange(0, len(cities)))
 def __init__(self, data):

 if type(data)is str:
data = self._read(data)
 elif type(data)is not np.ndarray or data.dtype != np.int32:
data = np.array(data, dtype=np.int32)
 self.data = data
if len(data)!= len(self.cities)+ 1:
raise Exception('Bad length')
 if data[0] != 0 or data[-1] != 0:
raise Exception('Must start/end with 0');
try:
x = np.zeros(len(data)- 1, dtype=np.bool)
 x[data[1:]] = 1
if np.min(x)== 0:
raise Exception('Repeated/missing cities')
 except IndexError:
raise Exception('Indexes out of bounds')
 @classmethod
def _read(cls, filename):
data = open(filename, 'r' ).read()
if data.startswith('Path'):
return pd.read_csv(io.StringIO(data)).Path.values
offs = data.find('TOUR_SECTION
')
 if offs != -1:
data = np.fromstring(data[offs+13:], sep='
', dtype=np.int32)
 data[-1] = 1
return data - 1
else:
data = data.replace('
', ' ')
 data = np.fromstring(data, sep=' ', dtype=np.int32)
 if len(data)!= data[0] + 1:
raise Exception('Unrecognized format in %s' % filename)
 return np.concatenate(( data[1:], [0]))
def info(self):
dist = np.abs(np.diff(self.coords[self.data]))
penalty = 0.1 * np.sum(dist[9::10] * self.penalized[self.data[9:-1:10]])
 dist = np.sum(dist)
 return { 'score': dist + penalty, 'dist': dist, 'penalty': penalty }
def dist(self):
return self.info() ['dist']
def score(self):
return self.info() ['score']
def __repr__(self):
return 'Tour: %s' % str(self.info())
 def to_csv(self, filename):
pd.DataFrame({'Path': self.data} ).to_csv(filename, index=False)
 def reversed(self):
return Tour(self.data[::-1])
 def roll(self, k):
return Tour(np.concatenate(( [0], np.roll(self.data[1:-1], k), [0])))
 def plot(self, cmap=mpl.cm.gist_rainbow, figsize=(25, 20)) :
fig, ax = plt.subplots(figsize=figsize)
 n = len(self.data)
 for i in range(201):
ind = self.data[n//200*i:min(n, n//200*(i+1)+1)]
ax.plot(self.cities.X[ind], self.cities.Y[ind], color=cmap(i/200.0), linewidth=1)
 ax.plot(self.cities.X[0], self.cities.Y[0], marker='*', markersize=15, markerfacecolor='k')
 ax.autoscale(tight=True)
 mpl.colorbar.ColorbarBase(ax=fig.add_axes([0.125, 0.075, 0.775, 0.01]),
norm=mpl.colors.Normalize(vmin=0, vmax=n),
cmap=cmap, orientation='horizontal' )<load_from_csv>
X_train, X_devtest, y_train, y_devtest = train_test_split(X_train, y_train, test_size=0.4)
 X_devtest, X_testest, y_devtest, y_testest = train_test_split(X_devtest, y_devtest, test_size=0.5)
 X_train = X_train.astype('float16')
 X_devtest = X_devtest.astype('float16')
 X_testest = X_testest.astype('float16' )
filename = '.. /input/lkh-solver/submission.csv'
Tour(filename )<compute_test_metric>
%matplotlib inline
degree = 3
gamma = 1.4
C = 13.0
best_C = C
best_gamma = gamma
best_degree = degree
best_score = -1
best_model = None
hist = {'C':[], 'acc': [], 'gamma':[], 'degree':[]}
svm = SVC(C=C, kernel='rbf', gamma=gamma, degree=degree, max_iter=-1)
 svm = OneVsRestClassifier(svm, n_jobs=4)
 svm.fit(X_train, y_train)
 best_model = svm
best_score = svm.score(X_devtest, y_devtest)
tour = Tour(filename)
 print('Original: %.2f' % tour.score())
 print('reversed: %.2f' % tour.reversed().score())
 print('roll(6): %.2f' % tour.roll(6 ).score())
 print('------------')
 print('Improvement: %.2f' %(tour.score() - tour.roll(6 ).score()))
tour = tour.roll(6 )<save_to_csv>
print("{} {} {}".format(best_C, best_gamma, best_score))
print("Train accuracy {}".format(best_model.score(X_train, y_train)))
 print("Cross validation accuracy {}".format(best_model.score(X_devtest, y_devtest)))
 print("Holdout test accuracy {}".format(best_model.score(X_testest, y_testest)) )
tour.to_csv('submission.csv')
 tour.plot()<define_variables>
y_test = best_model.predict(X_test )
seed = 2357
random.seed(seed )<load_from_csv>
y_pred = cuisine_encoder.inverse_transform(y_test )
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )<load_pretrained>
ids = [recipe['id'] for recipe in testing_data]
index_f_data = np.load('.. /input/split-test-dataset/index_of_fake_data.npy')
 index_r_data = np.load('.. /input/split-test-dataset/index_of_real_data.npy' )<load_from_csv>
with open('submission.csv','w')as file:
file.write('id,cuisine
')
 for id_,cuisine in zip(ids, y_pred):
file.write("{},{}
".format(id_, cuisine))