Kaggle competition: What's Cooking?
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?<import_modules>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn
import os
import json
import re
import nltk
import zipfile
from datetime import datetime
from sklearn.preprocessing import LabelEncoder
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import confusion_matrix, accuracy_score
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertTokenizer, BertModel
from scipy.stats import spearmanr<choose_model_class>
for t in ['train','test']:
with zipfile.ZipFile(".. /input/whats-cooking/{}.json.zip".format(t),"r")as z:
z.extractall(".")
 with open('./train.json')as data_file:
data = json.load(data_file)
 with open('./test.json')as test_file:
test = json.load(test_file )
class Model(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/bert-base-uncased-huggingface')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output = nn.Linear(768 * 2 + 3, 30)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks, q_a_len):
q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze(1 ).squeeze(1)
 a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze(1 ).squeeze(1)
 x = torch.cat(( q_x, a_x, q_a_len), -1)
 x = torch.sigmoid(self.output(x))
return x<import_modules>
df = pd.DataFrame(data)
 test_df = pd.DataFrame(test)
 test_ids = test_df['id']
df.head()
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset, TensorDataset
from sklearn.model_selection import train_test_split
from transformers import AdamW, get_linear_schedule_with_warmup<load_pretrained>
( df.isnull().sum() / len(df)) *100
data = torch.load('./t7/data.t7' )<load_pretrained>
( test_df.isnull().sum() / len(test_df)) *100
test_data = torch.load('./t7/test.t7' )<prepare_x_and_y>
def preprocess_df(df):
def process_string(x):
x = [" ".join([WordNetLemmatizer().lemmatize(q)for q in p.split() ])for p in x]
x = list(map(lambda x: re.sub(r'\ (.*oz.\)|crushed|crumbles|ground|minced|powder|chopped|sliced','', x), x))
x = list(map(lambda x: re.sub("[^a-zA-Z]", " ", x), x))
x = " ".join(x)
 x = x.lower()
return x
df = df.drop('id',axis=1)
 df['ingredients'] = df['ingredients'].apply(process_string)
 return df
test_set = TensorDataset(torch.LongTensor(np.array(test_data['txt'])) ,
torch.FloatTensor(np.array(test_data['num'])) )<load_pretrained>
def get_cuisine_cumulated_ingredients(df):
cuisine_df = pd.DataFrame(columns=['ingredients'])
 for cus in cuisine:
st = ""
for x in df[df.cuisine == cus]['ingredients']:
st += x
st += " "
cuisine_df.loc[cus,'ingredients'] = st
cuisine_df = cuisine_df.reset_index()
cuisine_df = cuisine_df.rename(columns ={'index':'cuisine'})
 return cuisine_df
test_loader = DataLoader(
 test_set,
batch_size=1,
shuffle=False )<choose_model_class>
df = preprocess_df(df)
 test_df = preprocess_df(test_df)
 cuisine_df = get_cuisine_cumulated_ingredients(df )
criterion = nn.BCELoss()<compute_test_metric>
train = df['ingredients']
target = df['cuisine']
test = test_df['ingredients']
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )<choose_model_class>
def count_vectorizer(train, test=None):
cv = CountVectorizer()
train = cv.fit_transform(train)
 if test is not None:
test = cv.transform(test)
 return train, test, cv
else:
return train, cv
gkf = GroupKFold(n_splits=5 ).split(X=train.question_body, groups=train.question_body)
 final_predicts = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 1, 2]:
model = Model()
model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=2e-5)
 train_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[train_idx]),\
torch.FloatTensor(np.array(data['num'])[train_idx]), \
torch.FloatTensor(np.array(data['label'])[train_idx]))
dev_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[valid_idx]),\
torch.FloatTensor(np.array(data['num'])[valid_idx]), \
torch.FloatTensor(np.array(data['label'])[valid_idx]))
train_loader = DataLoader(
 train_set,
batch_size=6,
shuffle=True, drop_last=True)
 dev_loader = DataLoader(
 dev_set,
batch_size=min(len(dev_set), 1),
shuffle=False)
 for epoch_idx in range(3):
for batch_idx,(model_in, model_num_in, labels)in enumerate(train_loader):
model.train()
optimizer.zero_grad()
model_in = model_in.cuda()
model_num_in = model_num_in.cuda()
labels = labels.cuda()
model_in = Variable(model_in, requires_grad=False)
 model_num_in = Variable(model_num_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], \
model_in[:, 4], model_in[:, 5], model_num_in)
 labels = Variable(labels, requires_grad=False)
 labels = labels.transpose(0, 1)
 scores = scores.transpose(0, 1)
 losses = [criterion(score, label)for score, label in zip(scores, labels)]
loss = sum(losses)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
print("train epoch: {} loss: {}".format(epoch_idx, loss.item() /30))
torch.cuda.empty_cache()
model.eval()
pre_list = []
tru_list = []
with torch.no_grad() :
for model_in, model_num_in, labels in dev_loader:
model_in = model_in.cuda()
model_num_in = model_num_in.cuda()
model_in = Variable(model_in, requires_grad=False)
 model_num_in = Variable(model_num_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], \
model_in[:, 3], model_in[:, 4], model_in[:, 5], model_num_in)
 pre_list.append(scores)
 tru_list.append(labels)
 dev_predicts = [pre.squeeze(0 ).cpu().numpy().tolist() for pre in pre_list]
truthes = [t.squeeze(0 ).numpy().tolist() for t in tru_list]
dev_rho = compute_spearmanr_ignore_nan(dev_predicts, truthes)
 print("dev score: ", dev_rho)
 torch.cuda.empty_cache()
model.eval()
test_predicts = []
with torch.no_grad() :
for model_in, model_num_in in test_loader:
model_in = model_in.cuda()
model_num_in = model_num_in.cuda()
model_in = Variable(model_in, requires_grad=False)
 model_num_in = Variable(model_num_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], \
model_in[:, 3], model_in[:, 4], model_in[:, 5], model_num_in)
 test_predicts.append(scores.reshape(scores.shape[-1]))
final_predicts.append(test_predicts)
 torch.cuda.empty_cache()<groupby>
def tfidf_vectorizer(train, test=None):
tfidf = TfidfVectorizer(stop_words='english',
ngram_range =(1 , 1),analyzer="word",
max_df =.57 , binary=False , token_pattern=r'\w+' , sublinear_tf=False)
 train = tfidf.fit_transform(train)
 if test is not None:
test = tfidf.transform(test)
 return train, test, tfidf
else:
return train, tfidf
pres = np.average(final_predicts, axis=0 )<define_variables>
train_tfidf, test_tfidf, tfidf = tfidf_vectorizer(train,test)
 cuisine_data_tfidf, cuisine_tfidf = tfidf_vectorizer(cuisine_df['ingredients'] )
test_output = [[p.item() for p in pre] for pre in pres]<define_variables>
red_cuisine_pca, cus_pca, var_cus_pca = get_PCA(( cuisine_data_tfidf ).toarray() ,2 )
output_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<data_type_conversions>
%%time
wcss_pca = get_kmeans_wcss(red_cuisine_pca,20 )
output_values = np.transpose(test_output ).tolist()<define_variables>
cluster_cus_pca, km_cus_pca = kmeans(red_cuisine_pca,3)
 cluster_cus_pca
output_dict = {k: v for k, v in zip(output_cols, output_values)}<data_type_conversions>
%%time
wcss = get_kmeans_wcss(train_tfidf,30 )
output_dict['qa_id'] = sample_submission['qa_id'].values.tolist()<create_dataframe>
cluster, km = kmeans(train_tfidf,19)
 cluster_test = km.predict(test_tfidf)
 cluster
output = pd.DataFrame.from_dict(output_dict )<define_variables>
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(cluster.reshape(-1, 1))
cluster_encoded = enc.transform(cluster.reshape(-1, 1)).toarray()
order = ['qa_id', 'question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<drop_column>
cluster_test_encoded = enc.transform(cluster_test.reshape(-1, 1)).toarray()
output = output[order]<save_to_csv>
train_tfidf_nonsparse = np.append(( train_tfidf ).toarray() , cluster_encoded, axis=1 )
output.to_csv('submission.csv', index=False )<import_modules>
test_tfidf_nonsparse = np.append(( test_tfidf ).toarray() , cluster_test_encoded, axis=1 )
from transformers import BertTokenizer, BertModel
from transformers import XLNetModel, XLNetTokenizer
from sklearn import preprocessing
from sklearn.model_selection import GroupKFold
import re
import pandas as pd<load_from_csv>
train = train_tfidf
test = test_tfidf
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv('.. /input/google-quest-qa-add-on/test_v3.csv')
 train = pd.read_csv(".. /input/google-quest-qa-add-on/train_v3.csv" )<define_variables>
param_grid = {'C': [0.001, 0.1, 1, 10, 50, 100, 500, 1000, 5000],
'penalty': ['l1','l2'],
'loss': ['hinge','squared hinge']}
grid = GridSearchCV(LinearSVC() , param_grid, refit = True, verbose = 3, n_jobs=-1, scoring='f1_micro' )
l_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<define_variables>
%%time
grid.fit(train, target )
columns = ['question_title', 'question_cleaned', 'answer_cleaned', 'category',
'question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<count_missing_values>
grid.best_params_
train[train.isnull().T.any() ]<categorify>
grid.best_score_
def txt_re(content):
res_list = []
for txt in content:
if pd.isnull(txt):
res_list.append('code')
 else:
txt = txt.strip()
txt = re.sub('https?.*$', '', txt)
 txt = re.sub('https?.*\s', '', txt)
 txt = re.sub('
+', '', txt)
 txt = re.sub('\r+', '', txt)
 res_list.append(txt)
 return res_list<define_variables>
def evalfn(C, gamma):
s = SVC(C=float(C), gamma=float(gamma), kernel='rbf', class_weight='balanced')
 f = cross_val_score(s, train, target, cv=5, scoring='f1_micro')
 return f.max()
content = []
labels = []
for col in zip(train[columns].values.tolist()):
cont = col[0][:4]
cont = txt_re(cont)
 label = col[0][4:]
content.append(cont)
 labels.append(label )<string_transform>
new_opt = BayesianOptimization(evalfn, {'C':(0.1, 1000),
'gamma':(0.0001, 1)} )
test_content = []
for col in zip(test[['question_title', 'question_body', 'answer', 'category']].values.tolist()):
cont = col[0][:4]
cont = txt_re(cont)
 test_content.append(cont )<load_pretrained>
C = 604.5300203551828
gamma = 0.9656489284085462
clf = SVC(C=float(C), gamma=float(gamma), kernel='rbf' )
tokenizer = BertTokenizer.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-tokenizer' )<categorify>
%%time
clf.fit(train, target )
q_inputs = []
q_input_masks = []
q_segment_masks = []
a_inputs = []
a_input_masks = []
a_segment_masks = []
for cont in content:
q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(q_input['input_ids'])== len(q_input['token_type_ids'])== len(q_input['attention_mask'])
 q_inputs.append(q_input['input_ids'])
 q_segment_masks.append(q_input['token_type_ids'])
 q_input_masks.append(q_input['attention_mask'])
 a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(a_input['input_ids'])== len(a_input['token_type_ids'])== len(a_input['attention_mask'])
 a_inputs.append(a_input['input_ids'])
 a_segment_masks.append(a_input['token_type_ids'])
 a_input_masks.append(a_input['attention_mask'] )<categorify>
now = datetime.now()
print("MODEL SAVED AT {}".format(now))
model_name = "SVC-whats-cooking-trial-final2-{}.pickle.dat".format(now)
 pickle.dump(clf, open(model_name, "wb"))
t_q_inputs = []
t_q_input_masks = []
t_q_segment_masks = []
t_a_inputs = []
t_a_input_masks = []
t_a_segment_masks = []
for cont in test_content:
t_q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_q_input['input_ids'])== len(t_q_input['token_type_ids'])== len(t_q_input['attention_mask'])
 t_q_inputs.append(t_q_input['input_ids'])
 t_q_segment_masks.append(t_q_input['token_type_ids'])
 t_q_input_masks.append(t_q_input['attention_mask'])
 t_a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_a_input['input_ids'])== len(t_a_input['token_type_ids'])== len(t_a_input['attention_mask'])
 t_a_inputs.append(t_a_input['input_ids'])
 t_a_segment_masks.append(t_a_input['token_type_ids'])
 t_a_input_masks.append(t_a_input['attention_mask'] )<categorify>
y_pred = clf.predict(test )
text = [[q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask]\
for q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask in \
zip(q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks)]<define_variables>
my_submission = pd.DataFrame({'id':test_ids})
 my_submission['cuisine'] = y_pred
now = datetime.now()
my_submission.to_csv('submission_{}.csv'.format(now), index=False)
 print('Saved file to disk as submission_{}.csv.'.format(now))