Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<load_from_csv>
import os
import json
import re
import pandas as pd
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import SVC
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import FunctionTransformer
db_size = pd.read_sql('SELECT page_count * page_size as size FROM pragma_page_count() , pragma_page_size() ', conn)['size'][0]
print(f'Total size of database is: {db_size/1e9:0.3f} GB' )<categorify>
def get_replacements() :
return {'wasabe': 'wasabi', '-': '', 'sauc': 'sauce',
'baby spinach': 'babyspinach', 'coconut cream': 'coconutcream',
'coriander seeds': 'corianderseeds', 'corn tortillas': 'corntortillas',
'cream cheese': 'creamcheese', 'fish sauce': 'fishsauce',
'purple onion': 'purpleonion','refried beans': 'refriedbeans',
'rice cakes': 'ricecakes', 'rice syrup': 'ricesyrup',
'sour cream': 'sourcream', 'toasted sesame seeds': 'toastedsesameseeds',
'toasted sesame oil': 'toastedsesameoil', 'yellow onion': 'yellowonion'}
def select_state(batch_cols, records):
return f<categorify>
def tranform_to_single_string(ingredients, lemmatizer, replacements, stop_pattern):
ingredients_text = ' '.join(iter(ingredients))
for key, value in replacements.items() :
ingredients_text = ingredients_text.replace(key, value)
 words = []
for word in ingredients_text.split() :
if not stop_pattern.match(word)and len(word)> 2:
word = lemmatizer.lemmatize(word)
 words.append(word)
 return ' '.join(words )
def update_state(df):
def get_select_params(r):
values_uc = f'({r.user_id}, {r.content_id}, {r.answered_correctly}, {1-r.answered_correctly})'
values_u = f'({r.user_id}, {r.answered_correctly}, {1-r.answered_correctly})'
return values_uc, values_u
values = df.apply(get_select_params, axis=1, result_type='expand')
 return f<feature_engineering>
def get_estimator() :
return SVC(C=300,
kernel='rbf',
gamma=1.5,
shrinking=True,
tol=0.001,
cache_size=1000,
class_weight=None,
max_iter=-1,
decision_function_shape='ovr',
random_state=42 )
%%time
df_batch_prior = None
counter = 0
for test_batch in iter_test:
counter += 1
if df_batch_prior is not None:
answers = eval(test_batch[0]['prior_group_answers_correct'].iloc[0])
 df_batch_prior['answered_correctly'] = answers
cursor.executescript(update_state(df_batch_prior[df_batch_prior.content_type_id == 0]))
if not counter % 100:
conn.commit()
df_batch_prior = test_batch[0][batch_cols_prior].astype({k: dtypes[k] for k in batch_cols_prior})
 df_batch = test_batch[0][test_batch[0].content_type_id == 0]
records = df_batch[batch_cols].fillna(0 ).to_records(index=False)
 df_batch = pd.read_sql(select_state(batch_cols, records), conn)
 predictions = model.predict(xgb.DMatrix(df_batch[test_cols]))
df_batch['answered_correctly'] = predictions
env.predict(df_batch[['row_id', 'answered_correctly']] )<import_modules>
def show_unique_ingredients(train):
ingredients = {}
for idx, row in train.iterrows() :
for ingredient in row['ingredients']:
if ingredient not in ingredients:
ingredients[ingredient] = {'sum': 0}
previous = ingredients[ingredient][row['cuisine']] if row['cuisine'] in ingredients[ingredient] else 0
ingredients[ingredient][row['cuisine']] = 1 + previous
ingredients[ingredient]['sum'] += 1
for ingredient in sorted(ingredients):
for cuisine in sorted(ingredients[ingredient], key=ingredients[ingredient].get, reverse=True):
print(f'{ingredient}:{cuisine}:{ingredients[ingredient][cuisine]}' )
from sklearn.metrics import roc_auc_score
from fastai.tabular.all import *
import gc
import fastai
from tqdm.notebook import tqdm as tqdm<define_variables>
def preprocess(train, test):
lemmatizer = WordNetLemmatizer()
replacements = get_replacements()
train['ingredients'] = train['ingredients'].apply(lambda x: list(map(lambda y: y.lower() , x)))
 test['ingredients'] = test['ingredients'].apply(lambda x: list(map(lambda y: y.lower() , x)))
 stop_pattern = re.compile('[\d’%]')
 transform = lambda ingredients: tranform_to_single_string(ingredients, lemmatizer, replacements, stop_pattern)
 train['x'] = train['ingredients'].apply(transform)
 test['x'] = test['ingredients'].apply(transform)
 vectorizer = make_pipeline(
 TfidfVectorizer(sublinear_tf=True),
FunctionTransformer(lambda x: x.astype('float'), validate=False)
)
 x_train = vectorizer.fit_transform(train['x'].values)
 x_train.sort_indices()
x_test = vectorizer.transform(test['x'].values)
 return x_train, x_test
<load_from_disk><EOS> 
%%time
def main() :
train = pd.read_json('.. /input/train.json')
 test = pd.read_json('.. /input/test.json')
 train['num_ingredients'] = train['ingredients'].apply(lambda x: len(x))
test['num_ingredients'] = test['ingredients'].apply(lambda x: len(x))
train = train[train['num_ingredients'] > 2]
x_train, x_test = preprocess(train, test)
 estimator = get_estimator()
y_train = train['cuisine'].values
classifier = OneVsRestClassifier(estimator, n_jobs=-1)
 classifier.fit(x_train, y_train)
 test['cuisine'] = classifier.predict(x_test)
 test[['id', 'cuisine']].to_csv('submission.csv', index=False)
 main()