Kaggle competition: What's Cooking? (Kernels Only)
def initial() :
df = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv')
 df['Z'] = 1 +.1 * ~df['CityId'].apply(sp.isprime)
 data = df[['X', 'Y', 'Z']].values
tour = np.loadtxt('.. /input/traveling-santa-lkh-solution/pure1502650.csv',
skiprows=1, dtype=int)
 return tour, data
tour, data = initial()<compute_test_metric>
random_state = 7
def read_dataset(path):
return json.load(open(path, encoding='utf-8'))
def distance(tour, data, pen=9):
xy, z = np.hsplit(data[tour], [2])
 dist = np.hypot(*(xy[:-1] - xy[1:] ).T)
 dist[pen::10] *= z[:-1][pen::10].flat
return dist
dist = distance(tour, data)
 dist<compute_test_metric>
print("Read Dataset...")
 df_train_raw=pd.read_json(os.path.join(challenge_path,'train.json')).set_index('id')
 df_train_raw['invalid']=[False] *(len(df_train_raw))
x_field='ingredients'
y_field='cuisine'
df_train_raw=df_train_raw.loc[df_train_raw.cuisine !=None]
df_train_raw.head()
distance(tour[3:9], data, pen = 9 - 3 % 10)== dist[3:8]<compute_test_metric>
lemmatizer = WordNetLemmatizer()
def preprocess(ingredients):
ingredients_text = ' '.join(ingredients)
 ingredients_text = ingredients_text.lower()
ingredients_text = ingredients_text.replace('-', ' ')
 words = []
for word in ingredients_text.split() :
if re.findall('[0-9]', word): continue
if len(word)<= 2: continue
if '’' in word: continue
word = lemmatizer.lemmatize(word)
 if len(word)> 0: words.append(word)
 return ' '.join(words)
 ing=df_train_raw[x_field].values.tolist()
cou=df_train_raw[y_field].values.tolist()
find_duplicated=False
dup=0
texts=[]
for i in range(len(ing)) :
if df_train_raw['invalid'].values[i]==False:
if len(ing[i])<1 or pd.isna(cou[i]):
df_train_raw['invalid'].values[i]=True
elif find_duplicated:
if ing[i] in ing[i+1:]:
dup+=1
print("found dup at {}".format(i))
df_train_raw['invalid'].values[i]=True
newtext=preprocess(df_train_raw[x_field].values[i])
 texts.append(newtext)
 df_train_raw['texts']=texts
f'Initial Score: {np.sum(dist):.2f}'<find_best_params>
len_orig=[len(x)for x in df_train_raw[x_field].values.tolist() ]
len_proc=[len(x.split(' ')) for x in texts]
df_train_raw['len_orig']=len_orig
df_train_raw['len_proc']=len_proc
df_train_raw.head()
def candidates(data, opt, ext):
nns = NearestNeighbors(n_neighbors=opt + ext ).fit(data[:, :2])
 kne = nns.kneighbors(data[:, :2], return_distance=False)
 np.random.shuffle(kne)
 cand = set()
for i in kne:
for j in combinations(i[1:], opt - 1):
cand.add(tuple(sorted(( i[0],)+ j)))
 return cand<find_best_params>
df_train=df_train_raw.loc[df_train_raw.invalid == False]
dup,len(df_train),len(df_train_raw )
list(candidates(data, opt=3, ext=0)) [:5]<compute_test_metric>
df_submission = pd.read_json(os.path.join(challenge_path,'test.json')).set_index('id')
 df_submission.head()
len(candidates(data, opt=2, ext=0))<compute_test_metric>
y_j=df_train[y_field].values
x_j=df_train['texts'].values
for i in range(len(df_submission)) :
df_submission[x_field].values[i]=preprocess(df_submission[x_field].values[i])
 x_j_sub = df_submission[x_field].values
len(candidates(data, opt=2, ext=1))<categorify>
y_lbr=LabelBinarizer()
y_j_conv=y_lbr.fit_transform(y_j)
 y_classes=y_lbr.classes_
y_classes_transformed=y_lbr.transform(y_classes )
def alternatives(tour, cuts, fil):
edges = [tuple(x)for x in np.split(tour, cuts)[1:-1]]
a, b = tour[cuts[0] - 1], tour[cuts[-1]]
alter = set()
for i in set(product(*zip(edges, [x[::-1] for x in edges]))):
for j in permutations(i):
if not fil or all(x != y for x, y in zip(edges, j)) :
alter.add(tuple(chain(( a,), *j,(b,))))
alter.discard(tuple(chain(( a,), *edges,(b,))))
return alter<define_search_space>
class_weights = class_weight.compute_class_weight('balanced', y_classes, y_j)
 class_weights={i:class_weights[i] for i in range(len(y_lbr.classes_)) }
class_weights
alternatives(tour, cuts = [2,5,7], fil=False )<define_search_space>
def tokenize(text):
tokens = nltk.word_tokenize(text)
 return tokens
x_cvr = TfidfVectorizer(analyzer='word', tokenizer=tokenize, token_pattern=None, stop_words='english')
 x_cvr.fit(x_j)
 x_analyze = x_cvr.build_analyzer()
x_j_conv =x_cvr.transform(x_j ).toarray()
x_j_sub_conv=x_cvr.transform(x_j_sub ).toarray()
alternatives(tour, cuts = [2,5,7], fil=True )<save_to_csv>
xt,xv,yt,yv,yjt,yjv=train_test_split(x_j_conv,y_j_conv,y_j, test_size=test_size, stratify=y_j, random_state=7, shuffle=True )
def submit(tour):
np.savetxt('submission.csv', tour, fmt='%d', header='Path', comments='' )<compute_test_metric>
K.clear_session()
np.random.seed(7)
 do=0.50
i = Input(shape=xt.shape[1:])
 x=Dropout(do-0.1 )(i)
 x=Dense(640 )(x)
 x=BatchNormalization()(x)
 x=Activation('relu' )(x)
 x=Dropout(do )(x)
 o=Dense(yt.shape[1],activation='linear' )(x)
 o=Dense(yt.shape[1],activation='softmax' )(o)
 model=Model(inputs=[i,], outputs=[o,])
 model.summary()
def kopt(opt, ext, fil):
tour, data = initial()
sequ = 1 + np.argsort(tour[1:])
 dist = distance(tour, data)
 print(f'opt:{opt} & ext:{ext} & fil:{fil}...')
 cand = candidates(data, opt, ext)
 print(f' Initial Score:\t{np.sum(dist):0.2f}')
 for c in cand:
cuts = sorted(sequ[j] for j in c)
 alter = alternatives(tour, cuts, fil)
 if not alter:
continue
atour, pen = np.array(list(alter)) , 9 -(cuts[0] - 1)% 10
adist = np.array([distance(x, data, pen)for x in atour])
 if np.any(np.sum(adist, 1)< np.sum(dist[cuts[0] - 1:cuts[-1]])) :
arg = np.argmin(np.sum(adist, 1))
dist[cuts[0] - 1:cuts[-1]] = adist[arg]
tour[cuts[0]:cuts[-1]] = atour[arg][1:-1]
sequ[atour[arg][1:-1]] = range(cuts[0], cuts[-1])
 print(f' Final Score:\t{np.sum(dist):0.2f}')
 submit(tour )<feature_engineering>
lr=0.005
epochs=100
verbose=2
model.compile(loss="categorical_crossentropy", optimizer=Adam(lr=lr), metrics=["accuracy"])
 mi=''
if mode=="train":
early_stopping = EarlyStopping(monitor='val_loss', patience=33, verbose=1)
 model_checkpoint2 = ModelCheckpoint("./best_loss{}.model".format(mi),monitor='val_loss',
save_best_only=True, verbose=1)
 model_checkpoint3 = ModelCheckpoint("./best_acc{}.model".format(mi),monitor='val_acc',
save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.3, patience=7, min_lr=0.00001, verbose=verbose)
 model.fit(x=[xt,],y=[yt]*1,validation_data=[[xv,],[yv]*1],callbacks=[early_stopping, model_checkpoint2, model_checkpoint3, reduce_lr], epochs=epochs, batch_size=256, shuffle=True, verbose=2)
 elif mode == 'commit':
early_stopping = EarlyStopping(monitor='loss', patience=19, verbose=1)
 model_checkpoint2 = ModelCheckpoint("./best_loss{}.model".format(mi),monitor='loss',
save_best_only=True, verbose=1)
 model_checkpoint3 = ModelCheckpoint("./best_acc{}.model".format(mi),monitor='acc',
save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.3, patience=5, min_lr=0.00001, verbose=verbose)
 model.fit(x=[x_j_conv,],y=[y_j_conv]*1,callbacks=[early_stopping, model_checkpoint2, model_checkpoint3, reduce_lr], epochs=epochs, batch_size=256, shuffle=True, verbose=2)
t0 = time.time()
kopt(opt=2, ext=0, fil=False)
 print(f'Time:\t{time.time() -t0:.2f}s' )<train_model>
if mode == 'train' or mode == 'commit':
model_loss=load_model(os.path.join('.',"best_loss.model"))
model_acc=load_model(os.path.join('.',"best_acc.model"))
elif mode=='submission':
model_loss=load_model(os.path.join(user_path,"best_loss.model"))
model_acc=load_model(os.path.join(user_path,"best_acc.model"))
print(model_loss.evaluate([xv,],[yv]*1))
print(model_acc.evaluate([xv,],[yv]*1))
t0 = time.time()
kopt(opt=2, ext=1, fil=False)
 print(f'Time:\t{time.time() -t0:.2f}s' )<train_model>
y_j_sub_conv=model_loss.predict(x_j_sub_conv,verbose=1)
 y_j_sub_conv2=model_acc.predict(x_j_sub_conv,verbose=1 )
t0 = time.time()
kopt(opt=3, ext=0, fil=False)
 print(f'Time:\t{time.time() -t0:.2f}s' )<feature_engineering>
y_j_sub=y_lbr.inverse_transform(y_j_sub_conv)
 y_j_sub2=y_lbr.inverse_transform(y_j_sub_conv2 )
t0 = time.time()
kopt(opt=4, ext=0, fil=True)
 print(f'Time:\t{time.time() -t0:.2f}s' )<import_modules>
df_submission[y_field]= y_j_sub
df_submission.head()
print("Generate Submission File...")
 sub=df_submission.drop(x_field,axis=1)
 sub.head(50)
 sub.to_csv('svm_output_loss.csv', index=True)
 sub.to_csv('submission.csv', index=True)
 with open('submission.csv', mode='r', encoding='utf-8')as f:
csvtext=f.read()
print(csvtext )
import numpy as np
import pandas as pd
import numba
from sympy import isprime, primerange
from math import sqrt
from sklearn.neighbors import KDTree
from tqdm import tqdm
from itertools import combinations, permutations
from functools import lru_cache<load_from_csv>
df_submission[y_field]= y_j_sub2
df_submission.head()
print("Generate Submission File...")
 sub=df_submission.drop(x_field,axis=1)
 sub.head(50)
 sub.to_csv('svm_output_acc.csv', index=True)
 sub.to_csv('submission.csv', index=True)
 with open('submission.csv', mode='r', encoding='utf-8')as f:
csvtext=f.read()
print(csvtext )