Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<set_options>
reload(sys)
 def k_to_one_hot(k_hot_vector):
classes = np.unique(k_hot_vector)
 one_hot_matrix = []
for i in np.arange(len(classes)) :
row =(k_hot_vector == classes[i] ).astype(int, copy = False)
 if len(one_hot_matrix)== 0:
one_hot_matrix = row
else:
one_hot_matrix = np.vstack(( one_hot_matrix, row))
return classes, one_hot_matrix.conj().transpose()
def read_data(filename):
with open(filename)as data_file:
data = json.load(data_file)
 ids, cuisines, ingredients = [], [], []
if 'cuisine' in data[0].keys() :
for i in range(len(data)) :
ids.append(data[i]['id'])
 cuisines.append(data[i]['cuisine'])
 ingredients.append(data[i]['ingredients'])
 else:
for i in range(len(data)) :
ids.append(data[i]['id'])
 ingredients.append(data[i]['ingredients'])
 return ids, cuisines, ingredients
def create_submission(test_ids, guess):
sub = np.transpose(np.vstack(( test_ids, guess)))
 sub = np.vstack(( ['id', 'cuisine'], sub))
sub_file_name = 'submission_' + str(datetime.now())[0:16] +'.csv'
sub_file_name = sub_file_name.replace(' ', '_')
 sub_file_name = sub_file_name.replace(':', '-')
 np.savetxt(sub_file_name, sub, delimiter=",", fmt="%s")
 return None
def remove_numbers(ing):
return [[re.sub("\d+", "", x)for x in y] for y in ing]
def remove_special_chars(ing):
ing = [[x.replace("-", " ")for x in y] for y in ing]
ing = [[x.replace("&", " ")for x in y] for y in ing]
ing = [[x.replace("'", " ")for x in y] for y in ing]
ing = [[x.replace("''", " ")for x in y] for y in ing]
ing = [[x.replace("%", " ")for x in y] for y in ing]
ing = [[x.replace("!", " ")for x in y] for y in ing]
ing = [[x.replace("(", " ")for x in y] for y in ing]
ing = [[x.replace(")", " ")for x in y] for y in ing]
ing = [[x.replace("/", " ")for x in y] for y in ing]
ing = [[x.replace("/", " ")for x in y] for y in ing]
ing = [[x.replace(",", " ")for x in y] for y in ing]
ing = [[x.replace(".", " ")for x in y] for y in ing]
ing = [[x.replace(u"\u2122", " ")for x in y] for y in ing]
ing = [[x.replace(u"\u00AE", " ")for x in y] for y in ing]
ing = [[x.replace(u"\u2019", " ")for x in y] for y in ing]
return ing
def make_lowercase(ing):
return [[x.lower() for x in y] for y in ing]
def remove_extra_whitespace(ing):
return [[re.sub('\s+', ' ', x ).strip() for x in y] for y in ing]
def stem_words(ing):
lmtzr = WordNetLemmatizer()
def word_by_word(strng):
return " ".join(["".join(lmtzr.lemmatize(w)) for w in strng.split() ])
 return [[word_by_word(x)for x in y] for y in ing]
def remove_units(ing):
remove_list = ['g', 'lb', 's', 'n']
def check_word(strng):
s = strng.split()
resw = [word for word in s if word.lower() not in remove_list]
return ' '.join(resw)
 return [[check_word(x)for x in y] for y in ing]
def extract_feats(ingredients, uniques):
feats_whole = np.zeros(( len(ingredients), len(uniques)))
 for i in range(len(ingredients)) :
for j in ingredients[i]:
feats_whole[i, uniques.index(j)] = 1
new_uniques = []
for m in uniques:
new_uniques.append(m.split())
 new_uniques = list(set(list(itertools.chain.from_iterable(new_uniques))))
feats_each = np.zeros(( len(ingredients), len(new_uniques)) ).astype(np.uint8)
 for i in range(len(ingredients)) :
for j in ingredients[i]:
for k in j.split() :
feats_each[i, new_uniques.index(k)] = 1
return np.hstack(( feats_whole, feats_each)).astype(bool)
 def load_model() :
mdl = Sequential()
mdl.add(Dense(512, init='glorot_uniform', activation='relu',
input_shape=(train_feats.shape[1],)))
 mdl.add(Dropout(0.5))
mdl.add(Dense(128, init='glorot_uniform', activation='relu'))
mdl.add(Dropout(0.5))
mdl.add(Dense(20, activation='softmax'))
mdl.compile(loss='categorical_crossentropy', optimizer='adagrad', metrics=['accuracy'])
 return mdl
if __name__ == '__main__':
print("
Preprocessing...
 ")
 train_ids, train_cuisines, train_ingredients = read_data(".. /input/train.json")
 train_ingredients = make_lowercase(train_ingredients)
 train_ingredients = remove_numbers(train_ingredients)
 train_ingredients = remove_special_chars(train_ingredients)
 train_ingredients = remove_extra_whitespace(train_ingredients)
 train_ingredients = remove_units(train_ingredients)
 train_ingredients = stem_words(train_ingredients)
 test_ids, test_cuisines, test_ingredients = read_data(".. /input/test.json")
 test_ingredients = make_lowercase(test_ingredients)
 test_ingredients = remove_numbers(test_ingredients)
 test_ingredients = remove_special_chars(test_ingredients)
 test_ingredients = remove_extra_whitespace(test_ingredients)
 test_ingredients = remove_units(test_ingredients)
 test_ingredients = stem_words(test_ingredients)
 print("Encoding...
 ")
 le = LabelEncoder()
targets = le.fit_transform(train_cuisines)
 classes, targets = k_to_one_hot(targets)
 print("Feature extraction...
 ")
 uniques = list(set([item for sublist in train_ingredients + test_ingredients for item in sublist]))
train_feats = extract_feats(train_ingredients, uniques)
 test_feats = extract_feats(test_ingredients, uniques)
 n_ensemble = 8
for ens in range(n_ensemble):
print("
\tTraining...", ens)
 model = load_model()
model_name = 'model' + str(ens)+ '.hdf5'
if os.path.isfile(model_name):
model.load_weights(model_name)
 model.fit(train_feats, targets, epochs=1000, batch_size=4096)
 model.save_weights(model_name, overwrite=True)
 preds = []
for ens in range(n_ensemble):
print("
Submission", ens)
 model = load_model()
model_name = 'model' + str(ens)+ '.hdf5'
model.load_weights(model_name)
 preds.append(model.predict_proba(test_feats))
print("
Predicting...")
 preds = sum(np.log(preds))
guess = le.inverse_transform(np.argmax(preds, axis=1))
create_submission(test_ids, guess)