Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<define_variables>
%matplotlib inline
init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print(os.listdir(".. /input"))
target = 'answered_correctly'<load_pretrained>
train_data = pd.read_json('.. /input/train.json')
 test_data = pd.read_json('.. /input/test.json' )
content_agg = pd.read_feather('/kaggle/input/riid-with-attempts-feather/content_agg_feather')
 content_agg.info()<load_from_csv>
print("The training data consists of {} recipes".format(len(train_data)) )
%%time
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1, 3, 4],
dtype={'question_id': 'int16', 'bundle_id': 'int16', 'part': 'int8', 'tags': 'object'} 
)
 tags = questions_df['tags'].str.split(' ', n=10, expand=True)
 tags.columns = ['tags1', 'tags2', 'tags3', 'tags4', 'tags5', 'tags6']
questions_df.drop(['tags'], axis=1, inplace=True)
 tags.drop(['tags1', 'tags4', 'tags5', 'tags6'], axis=1, inplace=True)
 questions_df = pd.concat([questions_df, tags], axis=1)
 questions_df['tags2'] = pd.to_numeric(questions_df['tags2'], errors='coerce',downcast='integer' ).fillna(-1)
 questions_df['tags3'] = pd.to_numeric(questions_df['tags3'], errors='coerce',downcast='integer' ).fillna(-1)
 questions_df['tags2'] = questions_df['tags2'].astype('uint16')
 questions_df['tags3'] = questions_df['tags3'].astype('uint16')
 train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True)
 del tags
gc.collect()
train_df.info()<define_variables>
print("Number of cuisine categories: {}".format(len(train_data.cuisine.unique())))
 train_data.cuisine.unique()
cat = ['part', 'tags2', 'tags3', 'bundle_id']
cont = ['user_count', 'user_correctness', 'content_count', 'content_correctness', 'prior_question_elapsed_time',
'attempts', 'hmean_user_content']
features = cat + cont<data_type_conversions>
def random_colours(number_of_colors):
colors = []
for i in range(number_of_colors):
colors.append("
return colors
def replace_na(x):
x = x.replace([np.inf, -np.inf], np.nan)
 x['tags2'].fillna(255, inplace=True)
 x['tags3'].fillna(255, inplace=True)
 return x.fillna(0 )<define_variables>
labelpercents = []
for i in train_data.cuisine.value_counts() :
percent =(i/sum(train_data.cuisine.value_counts())) *100
percent = "%.2f" % percent
percent = str(percent + '%')
 labelpercents.append(percent )
thresh = int(0.75 * len(train_df))
train_df = train_df.iloc[thresh:].reset_index(drop=True)
 train_range = list(range(len(train_df)- int(len(train_df)/ 10)))
 valid_range = list(range(len(train_df)- int(len(train_df)/ 10), len(train_df)))
 splits =(train_range, valid_range)
 gc.collect()
train_df.info()<create_dataframe>
print("Word Cloud Function.. ")
 stopwords = set(STOPWORDS)
 size =(20,10)
 def cloud(text, title, stopwords=stopwords, size=size):

 mpl.rcParams['figure.figsize']=(10.0,10.0)
 mpl.rcParams['font.size']=12
mpl.rcParams['savefig.dpi']=100
mpl.rcParams['figure.subplot.bottom']=.1
wordcloud = WordCloud(width=1600, height=800,
background_color='black',
stopwords=stopwords,
).generate(str(text))
fig = plt.figure(figsize=size, dpi=80, facecolor='k',edgecolor='k')
 plt.imshow(wordcloud,interpolation='bilinear')
 plt.axis('off')
 plt.title(title, fontsize=50,color='y')
 plt.tight_layout(pad=0)
 plt.show()
train_data["ing"] = train_data.ingredients.apply(lambda x: list(map(str, x)) , 1 ).str.join(' ')
 cloud(train_data["ing"].values, title="All Cuisine", size=[8,5] )
%%time
EPOCHS=5
BATCH_SIZE=2048
pd.options.mode.chained_assignment=None
to = TabularPandas(train_df, [Categorify, Normalize, FillMissing], cat, cont, target, splits=splits, inplace=True)
 dls = to.dataloaders(BATCH_SIZE )<train_on_grid>
train_data['seperated_ingredients'] = train_data['ingredients'].apply(','.join)
 test_data['seperated_ingredients'] = test_data['ingredients'].apply(','.join )
def my_auc(inp, targ):
"Simple wrapper around scikit's roc_auc_score function for regression problems"
inp,targ = flatten_check(inp,targ)
 return roc_auc_score(targ.cpu().numpy() , inp.cpu().numpy())
 learn = tabular_learner(dls, layers=[200,200], metrics=my_auc)
 lr_find_res = learn.lr_find()
learn.fit_one_cycle(EPOCHS, lr_find_res.lr_min / 10 )<predict_on_test>
import nltk
from collections import Counter
def predict_batch(self, df):
dl = self.dls.test_dl(df)
 dl.dataset.conts = dl.dataset.conts.astype(np.float32)
 inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
 return preds.numpy()
setattr(learn, 'predict_batch', predict_batch )<drop_column>
train_data['for ngrams']=train_data['seperated_ingredients'].str.replace(',',' ' )
del dls
del to
del train_df
gc.collect()<load_from_disk>
def generate_ngrams(text, n):
words = text.split(' ')
 iterations = len(words)- n + 1
for i in range(iterations):
yield words[i:i + n]
def net_diagram(*cuisines):
ngrams = {}
for title in train_data[train_data.cuisine==cuisines[0]]['for ngrams']:
for ngram in generate_ngrams(title, 2):
ngram = ','.join(ngram)
 if ngram in ngrams:
ngrams[ngram] += 1
else:
ngrams[ngram] = 1
ngrams_mws_df = pd.DataFrame.from_dict(ngrams, orient='index')
 ngrams_mws_df.columns = ['count']
ngrams_mws_df['cusine'] = cuisines[0]
ngrams_mws_df.reset_index(level=0, inplace=True)
 ngrams = {}
for title in train_data[train_data.cuisine==cuisines[1]]['for ngrams']:
for ngram in generate_ngrams(title, 2):
ngram = ','.join(ngram)
 if ngram in ngrams:
ngrams[ngram] += 1
else:
ngrams[ngram] = 1
ngrams_mws_df1 = pd.DataFrame.from_dict(ngrams, orient='index')
 ngrams_mws_df1.columns = ['count']
ngrams_mws_df1['cusine'] = cuisines[1]
ngrams_mws_df1.reset_index(level=0, inplace=True)
 cuisine1=ngrams_mws_df.sort_values('count',ascending=False)[:25]
cuisine2=ngrams_mws_df1.sort_values('count',ascending=False)[:25]
df_final=pd.concat([cuisine1,cuisine2])
 g = nx.from_pandas_edgelist(df_final,source='cusine',target='index')
 cmap = plt.cm.RdYlGn
colors = [n for n in range(len(g.nodes())) ]
k = 0.35
pos=nx.spring_layout(g, k=k)
 nx.draw_networkx(g,pos, node_size=df_final['count'].values*8, cmap = cmap, node_color=colors, edge_color='grey', font_size=15, width=3)
 plt.title("Top 25 Bigrams for %s and %s" %(cuisines[0],cuisines[1]), fontsize=30)
 plt.gcf().set_size_inches(30,30)
 plt.show()
plt.savefig('network.png' )
%%time
train_df = pd.read_feather('/kaggle/input/riid-with-attempts-feather/train_with_attempts')
 train_df = train_df.loc[train_df['answered_correctly'] != -1]
train_df.sort_values('timestamp', inplace=True)
 train_df.drop(['prior_question_had_explanation', 'prior_question_elapsed_time', 'timestamp', 'attempts',
'user_correctness', 'user_count', 'content_correctness', 'content_count', 'hmean_user_content'], axis=1, inplace=True)
 train_df.reset_index(drop=True, inplace=True)
 gc.collect()
train_df.info()<groupby>
features = []
for item in train_data['ingredients']:
features.append(item)
 features_test = []
for item in test_data['ingredients']:
features_test.append(item )
%%time
def user_states(df):
mean_user_accuracy = train_df.groupby('user_id')[target].mean().values
answered_correctly_user = train_df.groupby('user_id')[target].sum().values
answered_user = train_df.groupby('user_id')[target].count().values
state = dict()
for user_id in train_df['user_id'].sort_values().unique() :
state[user_id] = {}
total = len(state.keys())
 user_content = train_df.groupby('user_id')['content_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = train_df.groupby(['user_id', 'content_id'])[target].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array ).values
for user_id, content, attempt in tqdm(zip(state.keys() , user_content, user_attempts),total=total):
state[user_id]['user_content_attempts'] = dict(zip(content, attempt))
del user_content, user_attempts
gc.collect()
for idx, user_id in enumerate(state.keys()):
state[user_id]['user_correctness'] = mean_user_accuracy[idx]
state[user_id]['user_sum'] = answered_correctly_user[idx]
state[user_id]['user_count'] = answered_user[idx]
return state
user_states = user_states(train_df)
 del train_df
gc.collect()<split>
features_processed= []
for item in features:
newitem = []
for ingr in item:
ingr.lower()
ingr = re.sub("[^a-zA-Z]"," ",ingr)
 ingr = re.sub(( r'\b(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\b'), ' ', ingr)
 newitem.append(ingr)
 features_processed.append(newitem)
 features_test_processed= []
for item in features_test:
newitem = []
for ingr in item:
ingr.lower()
ingr = re.sub("[^a-zA-Z]"," ",ingr)
 ingr = re.sub(( r'\b(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\b'), ' ', ingr)
 newitem.append(ingr)
 features_test_processed.append(newitem )
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None<data_type_conversions>
count_m = []
for recipe in features_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed training sample: " + str(len(count_m)) )
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<feature_engineering>
count_m = []
for recipe in features_test_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed test sample: " + str(len(count_m)) )
for(test_df, sample_prediction_df)in iter_test:
y_preds = []
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_states[user_id]['user_sum'] += answered_correctly
user_states[user_id]['user_count'] += 1
user_states[user_id]['user_content_attempts'][content_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 user_content_attempts = np.zeros(len(test_df), dtype=np.int8)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
if user_id not in user_states.keys() :
user_states[user_id] = {'user_content_attempts':{user_id:0},
'user_correctness':0,
'user_count':0,
'user_sum':0}
if content_id not in user_states[user_id]['user_content_attempts'].keys() :
user_states[user_id]['user_content_attempts'][content_id] = 0
user_sum[i] = user_states[user_id]['user_sum']
user_count[i] = user_states[user_id]['user_count']
user_content_attempts[i] = user_states[user_id]['user_content_attempts'][content_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_correctness'] = user_sum / user_count
test_df['user_count'] = user_count
test_df['content_count'] = content_count
test_df['content_correctness'] = content_sum / content_count
test_df['attempts'] = user_content_attempts
test_df['hmean_user_content'] = 2 *(
(test_df['user_correctness'] * test_df['content_correctness'])/ 
(test_df['user_correctness'] + test_df['content_correctness'])
 ).astype('float32')
 test_df = replace_na(test_df)
 test_df[target] = learn.predict_batch(learn, test_df[features])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
vectorizer = CountVectorizer(analyzer = "word",
ngram_range =(1,1),
binary = True,
tokenizer = None,
preprocessor = None,
stop_words = None,
max_df = 0.99 )
import gc
import joblib
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
from numba import njit
from scipy.optimize import minimize, fsolve<define_variables>
train_X = vectorizer.fit_transform([str(i)for i in features_processed])
 test_X = vectorizer.transform([str(i)for i in features_test_processed] )
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
debug = False
validaten_flg = False<data_type_conversions>
target = train_data['cuisine']
def add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)) :
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1<categorify>
lb = LabelEncoder()
train_Y = lb.fit_transform(target )
feld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
if debug:
train = train[:1000000]
valid = valid[:10000]
train = train.loc[train.content_type_id == False].reset_index(drop=True)
 valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)
 content_df = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'] ).reset_index()
content_df.columns = ['content_id', 'answered_correctly_avg_c']
train = pd.merge(train, content_df, on=['content_id'], how="left")
 valid = pd.merge(valid, content_df, on=['content_id'], how="left")
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)
 valid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)
 prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()
train['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 questions_df = pd.read_csv(question_file)
 train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )<load_pretrained>
X_train, X_test, y_train, y_test = train_test_split(train_X, train_Y , random_state = 0 )
joblib.dump(answered_correctly_sum_u_dict, "answered_correctly_sum_u_dict.pkl.zip")
 joblib.dump(count_u_dict, "count_u_dict.pkl.zip")
 joblib.dump(prior_question_elapsed_time_mean, "prior_question_elapsed_time_mean.pkl.zip" )<categorify>
clfs = []
rfc = RandomForestClassifier(n_estimators=50, random_state=42, n_jobs=-1)
 rfc.fit(X_train, y_train)
 print('RFC LogLoss {score}'.format(score=log_loss(y_test, rfc.predict_proba(X_test))))
clfs.append(rfc)
 logreg = LogisticRegression(random_state = 42)
 logreg.fit(X_train, y_train)
 print('LogisticRegression LogLoss {score}'.format(score=log_loss(y_test, logreg.predict_proba(X_test))))
clfs.append(logreg)
 svc = SVC(random_state=42,probability=True, kernel='linear')
 svc.fit(X_train, y_train)
 print('SVC LogLoss {score}'.format(score=log_loss(y_test, svc.predict_proba(X_test))))
clfs.append(svc)
content_df.to_feather("content_df.feather")
 questions_df.to_feather("questions_df.feather" )<prepare_x_and_y>
predictions = []
for clf in clfs:
predictions.append(clf.predict_proba(X_test))
def log_loss_func(weights):

 final_prediction = 0
for weight, prediction in zip(weights, predictions):
final_prediction += weight*prediction
return log_loss(y_test, final_prediction)
 starting_values = [0.5]*len(predictions)
 cons =({'type':'eq','fun':lambda w: 1-sum(w)})
 bounds = [(0,1)]*len(predictions)
 res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons)
 print('Ensamble Score: {best_score}'.format(best_score=res['fun']))
print('Best Weights: {weights}'.format(weights=res['x']))
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _ = gc.collect()<create_dataframe>
vclf=VotingClassifier(estimators=[('clf1',RandomForestClassifier(n_estimators = 50,random_state = 42)) , 
('clf2',LogisticRegression(random_state = 42)) , 
('clf3',SVC(kernel='linear',random_state = 42,probability=True))
],
voting='soft', weights = [0.05607363, 0.70759724, 0.23632913])
 vclf.fit(train_X, train_Y )
lgb_train = lgb.Dataset(train[FEATS], y_tr)
 lgb_valid = lgb.Dataset(valid[FEATS], y_va)
 del train, y_tr
_ = gc.collect()<train_model>
kfold = model_selection.KFold(n_splits=5, random_state=42)
 valscores = model_selection.cross_val_score(vclf, train_X, train_Y, cv=kfold)
 print('Mean accuracy on 5-fold cross validation: ' + str(np.mean(valscores)) )
<save_model><EOS> 
predictions = vclf.predict(test_X)
 predictions = lb.inverse_transform(predictions)
 predictions_final = pd.DataFrame({'cuisine' : predictions , 'id' : test_data.id }, columns=['id', 'cuisine'])
 predictions_final.to_csv('Final_submission.csv', index = False )