Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<categorify>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from nltk import word_tokenize, download
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.stem import WordNetLemmatizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.ensemble import ExtraTreesClassifier
def setup_input(image, label, data, label_name):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}, label<categorify>
train = pd.read_json(".. /input/train.json")
 test = pd.read_json(".. /input/test.json")
 y_tr = train.cuisine
def get_training_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<categorify>
lemmatizer = WordNetLemmatizer()
def lemmatize_ingredients(df):
all_ingredients = set()
ingredients_list = []
for i in range(len(df.ingredients)) :
lemmatized_list = []
for ingr in df.ingredients[i]:
split_ingr = ingr.split(" ")
 lemmatized = []
for word in split_ingr:
lemmatized.append(lemmatizer.lemmatize(word.lower()))
all_ingredients.add(" ".join(lemmatized))
lemmatized_list.append(" ".join(lemmatized))
ingredients_list.append(lemmatized_list)
 df['ingredients_lem'] = ingredients_list
return df, all_ingredients
def get_validation_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset<create_dataframe>
train, all_ingredients = lemmatize_ingredients(train)
 test, _ = lemmatize_ingredients(test )
train_dataset = load_dataset(TRAIN_FNAMES['fold_1'])
 train_dataset<init_hyperparams>
train_tfidf_features = tfidf.fit_transform(arraytotext(train.ingredients_lem))
test_tfidf_features= tfidf.transform(arraytotext(test.ingredients_lem))
debug=0
histories = []
best_epoch={fn: np.zeros(NBEST)for fn in range(1, NFOLDS+1)}
best_score={fn: np.zeros(NBEST)for fn in range(1, NFOLDS+1)}
for fold_num in range(1, NFOLDS+1):
tf.keras.backend.clear_session()
tf.tpu.experimental.initialize_tpu_system(tpu)
 print("="*50)
 print(f"Starting fold {fold_num} out of {NFOLDS}...")
 files_trn=TRAIN_FNAMES[f"fold_{fold_num}"]
files_val=VAL_FNAMES[f"fold_{fold_num}"]
if debug:
NUMS = 2
files_trn=files_trn[0:2]
files_val=files_val[0:2]
EPOCHS=2
train_dataset = load_dataset(files_trn)
 train_dataset = train_dataset.map(setup_input, num_parallel_calls=AUTO)
 val_dataset = load_dataset(files_val, ordered = True)
 val_dataset = val_dataset.map(setup_input, num_parallel_calls=AUTO)
 model = get_model()
STEPS_PER_EPOCH = count_data_items(files_trn)// BATCH_SIZE
print(f'STEPS_PER_EPOCH = {STEPS_PER_EPOCH}')
 lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 history = model.fit(get_training_dataset(train_dataset),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback,
save_best_n(fold_num, model),
],
validation_data=get_validation_dataset(val_dataset),
class_weight=class_weights,
verbose=2, 
)
 idx_sorted=np.argsort(best_score[fold_num])
 best_score[fold_num]=np.array(best_score[fold_num])[idx_sorted]
best_epoch[fold_num]=np.array(best_epoch[fold_num])[idx_sorted]
print(f"
Fold {fold_num} is finished.The best epochs: {[int(best_epoch[fold_num][i])for i in range(len(best_epoch[fold_num])) ]}")
 print(f"The corresponding scores: {[round(best_score[fold_num][i], 5)for i in range(len(best_epoch[fold_num])) ]}")
 histories.append(history )<save_to_csv>
et = ExtraTreesClassifier(n_estimators=25, max_depth=300, min_samples_split=5, min_samples_leaf=1, random_state=None, min_impurity_decrease=1e-7 )
train.to_csv('train.csv' )<set_options>
model = OneVsRestClassifier(et)
 model.fit(train_tfidf_features, y_tr )
<create_dataframe><EOS> 
predictions = model.predict(test_tfidf_features)
 submission = pd.DataFrame()
submission['id'] = test.id
submission['cuisine'] = predictions
submission.to_csv('20180718_ova_etc_4.csv', index=False )