Kaggle competition: What's Cooking?
data = {'txt': text, 'label': labels}<categorify>
archive_train = zipfile.ZipFile("/kaggle/input/whats-cooking/train.json.zip",'r')
 archive_train
t_text = [[t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask] \
for t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask in \
zip(t_q_inputs, t_q_input_masks, t_q_segment_masks, t_a_inputs, t_a_input_masks, t_a_segment_masks)]<create_dataframe>
train_data = pd.read_json(archive_train.read('train.json'))
train_data
t_data = {'txt': t_text}<save_model>
archive_test = zipfile.ZipFile("/kaggle/input/whats-cooking/test.json.zip",'r')
 test_data = pd.read_json(archive_test.read("test.json"))
test_data
torch.save(data, './t7/data.t7' )<save_model>
train_data['cuisine'].value_counts().shape
torch.save(t_data, './t7/test.t7' )<import_modules>
train_ingr_count = {}
m = train_data.shape[0]
for i in range(m):
for j in train_data["ingredients"][i]:
if j in train_ingr_count.keys() :
train_ingr_count[j] += 1
else:
train_ingr_count[j] = 1
len(train_ingr_count )
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertTokenizer, BertModel
from scipy.stats import spearmanr<choose_model_class>
test_ingr_count = {}
m = test_data.shape[0]
for i in range(m):
for j in test_data["ingredients"][i]:
if j in test_ingr_count.keys() :
test_ingr_count[j] += 1
else:
test_ingr_count[j] = 1
len(test_ingr_count )
class Model(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-pytorch-model')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output = nn.Linear(768 * 2, 30)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks):
q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze()
a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze()
x = torch.cat(( q_x, a_x), -1)
 x = torch.sigmoid(self.output(x))
return x<import_modules>
train_ingr_missing = []
for i in test_ingr_count.keys() :
if i not in train_ingr_count.keys() :
train_ingr_missing.append(i)
 print(len(train_ingr_missing))
for i in train_ingr_missing:
train_ingr_count[i] = 0
print(len(train_ingr_count))
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset, TensorDataset
from sklearn.model_selection import train_test_split
from transformers import AdamW, get_linear_schedule_with_warmup<load_pretrained>
test_ingr_missing = []
for i in train_ingr_count.keys() :
if i not in test_ingr_count.keys() :
test_ingr_missing.append(i)
 print(len(test_ingr_missing))
for i in test_ingr_missing:
test_ingr_count[i] = 0
print(len(test_ingr_count))
data = torch.load('./t7/data.t7' )<load_pretrained>
for i in train_ingr_count.keys() :
train_data[i] = np.zeros(len(train_data))
test_data = torch.load('./t7/test.t7' )<create_dataframe>
for i in test_ingr_count.keys() :
test_data[i] = np.zeros(len(test_data))
test_set = TensorDataset(torch.LongTensor(np.array(test_data['txt'])) )<load_pretrained>
for i in range(len(train_data)) :
for j in train_data['ingredients'][i]:
train_data[j].iloc[i] = 1
test_loader = DataLoader(
 test_set,
batch_size=1,
shuffle=False )<compute_test_metric>
for i in range(len(test_data)) :
for j in test_data['ingredients'][i]:
test_data[j].iloc[i] = 1
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )<create_dataframe>
train_data.drop('ingredients',axis=1,inplace=True)
 test_data.drop('ingredients',axis=1,inplace=True )
gkf = GroupKFold(n_splits=5 ).split(X=train.question_body, groups=train.question_body)
 final_predicts = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 1, 2]:
model = Model()
model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=2e-5)
 train_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[train_idx]),\
torch.FloatTensor(np.array(data['label'])[train_idx]))
dev_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[valid_idx]),\
torch.FloatTensor(np.array(data['label'])[valid_idx]))
train_loader = DataLoader(
 train_set,
batch_size=6,
shuffle=True, drop_last=True)
 dev_loader = DataLoader(
 dev_set,
batch_size=min(len(dev_set), 1),
shuffle=False)
 for epoch_idx in range(3):
for batch_idx,(model_in, labels)in enumerate(train_loader):
model.train()
optimizer.zero_grad()
model_in = model_in.cuda()
labels = labels.cuda()
model_in = Variable(model_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], model_in[:, 4], model_in[:, 5])
 labels = Variable(labels, requires_grad=False)
 labels = labels.transpose(0, 1)
 scores = scores.transpose(0, 1)
 losses = [criterion(score, label)for score, label in zip(scores, labels)]
loss = sum(losses)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
print("train epoch: {} loss: {}".format(epoch_idx, loss.item() /30))
torch.cuda.empty_cache()
model.eval()
pre_list = []
tru_list = []
with torch.no_grad() :
for model_in, labels in dev_loader:
model_in = model_in.cuda()
model_in = Variable(model_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], model_in[:, 4], model_in[:, 5])
 pre_list.append(scores)
 tru_list.append(labels)
 dev_predicts = [[p.item() for p in pre] for pre in pre_list]
truthes = [t.squeeze(0 ).numpy().tolist() for t in tru_list]
dev_rho = compute_spearmanr_ignore_nan(dev_predicts, truthes)
 print("dev score: ", dev_rho)
 torch.cuda.empty_cache()
model.eval()
test_predicts = []
with torch.no_grad() :
for model_in in test_loader:
model_in = model_in[0].cuda()
model_in = Variable(model_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], model_in[:, 4], model_in[:, 5])
 test_predicts.append(scores)
 final_predicts.append(test_predicts)
 torch.cuda.empty_cache()<groupby>
test_data = test_data[train_data.drop('cuisine',axis=1 ).columns]
pres = np.average(final_predicts, axis=0 )<define_variables>
X = train_data.drop(['id','cuisine'],axis=1)
 y = train_data['cuisine']
X_train,X_test,y_train,y_test = train_test_split(X,y)
 print(X_test.shape,y_test.shape)
 print(X_train.shape,y_train.shape )
test_output = [[p.item() for p in pre] for pre in pres]<define_variables>
lr = LogisticRegression()
lr.fit(X_train,y_train)
 lr.score(X_test,y_test )
output_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<data_type_conversions>
pred = lr.predict(test_data.drop('id',axis=1))
output_values = np.transpose(test_output ).tolist()<define_variables>
submission = pd.DataFrame(data=pred,columns=['cuisine'])
 submission['id'] = test_data['id']
submission.set_index("id",inplace=True )
<data_type_conversions><EOS> 
submission.to_csv('submission.csv' )