Kaggle competition: What's Cooking?
test_v3 = pd.read_csv(".. /input/google-quest-qa-add-on/test_v3.csv")
 train_v3 = pd.read_csv(".. /input/google-quest-qa-add-on/train_v3.csv" )<define_variables>
archive_train=zipfile.ZipFile('/kaggle/input/whats-cooking/train.json.zip','r')
 archive_train
l_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<define_variables>
train_data=pd.read_json(archive_train.read('train.json'))
train_data.head()
columns = ['question_title', 'question_cleaned', 'answer_cleaned', 'category',
'question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<categorify>
archive_test=zipfile.ZipFile('/kaggle/input/whats-cooking/test.json.zip','r')
 test_data=pd.read_json(archive_test.read('test.json'))
test_data.head()
def txt_re(content):
res_list = []
for txt in content:
if pd.isnull(txt):
res_list.append('code')
 else:
txt = txt.strip()
txt = re.sub('https?.*$', '', txt)
 txt = re.sub('https?.*\s', '', txt)
 txt = re.sub('
+', '', txt)
 txt = re.sub('\r+', '', txt)
 txt = re.sub('&gt;', '>', txt)
 txt = re.sub('&lt;', '<', txt)
 txt = re.sub('&amp;', '&', txt)
 txt = re.sub('&quot;', '"', txt)
 res_list.append(txt)
 return res_list<define_variables>
train_data['cuisine'].value_counts()
content = []
labels = []
for col in zip(train_v3[columns].values.tolist()):
cont = col[0][:3]
cont = txt_re(cont)
 label = col[0][4:]
content.append(cont)
 labels.append(label )<string_transform>
train_ingredients_count={}
for i in range(len(train_data)) :
for j in train_data['ingredients'][i]:
if j in train_ingredients_count.keys() :
train_ingredients_count[j]+=1
else:
train_ingredients_count[j]=1
test_content = []
for col in zip(test_v3[['question_title', 'question_body', 'answer']].values.tolist()):
cont = col[0]
cont = txt_re(cont)
 test_content.append(cont )<load_pretrained>
test_ingredients_count={}
for i in range(len(test_data)) :
for j in test_data['ingredients'][i]:
if j in test_ingredients_count.keys() :
test_ingredients_count[j]+=1
else:
test_ingredients_count[j]=1
tokenizer = BertTokenizer.from_pretrained('.. /input/bert-base-uncased-huggingface' )<categorify>
ingredients_missing_train=[]
for i in test_ingredients_count.keys() :
if i not in train_ingredients_count.keys() :
ingredients_missing_train.append(i)
 train_ingredients_count[i]=0
print(len(ingredients_missing_train))
q_inputs = []
q_input_masks = []
q_segment_masks = []
a_inputs = []
a_input_masks = []
a_segment_masks = []
for cont in content:
q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(q_input['input_ids'])== len(q_input['token_type_ids'])== len(q_input['attention_mask'])
 q_inputs.append(q_input['input_ids'])
 q_segment_masks.append(q_input['token_type_ids'])
 q_input_masks.append(q_input['attention_mask'])
 a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(a_input['input_ids'])== len(a_input['token_type_ids'])== len(a_input['attention_mask'])
 a_inputs.append(a_input['input_ids'])
 a_segment_masks.append(a_input['token_type_ids'])
 a_input_masks.append(a_input['attention_mask'] )<define_variables>
ingredients_missing_test=[]
for i in train_ingredients_count.keys() :
if i not in test_ingredients_count.keys() :
ingredients_missing_test.append(i)
 test_ingredients_count[i]=0
print(len(ingredients_missing_test))
q_title_len = []
q_body_len = []
a_len = []
for col in zip(train_v3[['question_title', 'question_body', 'answer']].values.tolist()):
q_title_len.append(len(tokenizer.tokenize(col[0][0])))
 q_body_len.append(len(tokenizer.tokenize(col[0][1])))
 a_len.append(len(tokenizer.tokenize(col[0][2])) )<categorify>
for i in train_ingredients_count.keys() :
train_data[i]=np.zeros(len(train_data))
t_q_inputs = []
t_q_input_masks = []
t_q_segment_masks = []
t_a_inputs = []
t_a_input_masks = []
t_a_segment_masks = []
for cont in test_content:
t_q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_q_input['input_ids'])== len(t_q_input['token_type_ids'])== len(t_q_input['attention_mask'])
 t_q_inputs.append(t_q_input['input_ids'])
 t_q_segment_masks.append(t_q_input['token_type_ids'])
 t_q_input_masks.append(t_q_input['attention_mask'])
 t_a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_a_input['input_ids'])== len(t_a_input['token_type_ids'])== len(t_a_input['attention_mask'])
 t_a_inputs.append(t_a_input['input_ids'])
 t_a_segment_masks.append(t_a_input['token_type_ids'])
 t_a_input_masks.append(t_a_input['attention_mask'] )<define_variables>
for i in test_ingredients_count.keys() :
test_data[i]=np.zeros(len(test_data))
t_q_title_len = []
t_q_body_len = []
t_a_len = []
for col in zip(test_v3[['question_title', 'question_body', 'answer']].values.tolist()):
t_q_title_len.append(len(tokenizer.tokenize(col[0][0])))
 t_q_body_len.append(len(tokenizer.tokenize(col[0][1])))
 t_a_len.append(len(tokenizer.tokenize(col[0][2])) )<define_variables>
for i in range(len(train_data)) :
for j in train_data['ingredients'][i]:
train_data[j].iloc[i]=1
train_text = [[q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask]\
for q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask in \
zip(q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks)]<define_variables>
for i in range(len(test_data)) :
for j in test_data['ingredients'][i]:
test_data[j].iloc[i]=1
train_num = [[q_t, q_b, a] for q_t, q_b, a in zip(q_title_len, q_body_len, a_len)]<normalization>
test_data=test_data[train_data.drop('cuisine',axis=1 ).columns]
scaler = StandardScaler()
train_num = scaler.fit_transform(train_num ).tolist()<categorify>
X=train_data.drop(['id','ingredients','cuisine'],axis=1)
 y=train_data['cuisine']
test_text = [[t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask] \
for t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask in \
zip(t_q_inputs, t_q_input_masks, t_q_segment_masks, t_a_inputs, t_a_input_masks, t_a_segment_masks)]<define_variables>
Xtrain,Xval,ytrain,yval=train_test_split(X,y,random_state=42 )
test_num = [[q_t, q_b, a] for q_t, q_b, a in zip(t_q_title_len, t_q_body_len, t_a_len)]<normalization>
from sklearn.linear_model import LogisticRegression
test_num = scaler.transform(test_num ).tolist()<create_dataframe>
lr=LogisticRegression(solver='liblinear')
 lr.fit(Xtrain,ytrain )
data = {'txt': train_text, 'num': train_num, 'label': labels}<create_dataframe>
lr.score(Xval,yval )
t_data = {'txt': test_text, 'num': test_num}<save_model>
test_data['cuisine']=lr.predict(test_data.drop(['id','ingredients'],axis=1))
<save_model><EOS> 
sub=test_data[['id','cuisine']]
sub.set_index('id',inplace=True)
 sub.to_csv('subWCng.csv' )