Kaggle competition: Google QUEST Q&A Labeling
X_res, y_res = smote.fit_resample(x_smote, y_smote )<groupby>
import numpy as np
from transformers import BertTokenizer, BertModel
import re
import pandas as pd
merged_drug_to_targets = merged_drug.groupby(by = 'drug_id')[target_cols].max()<concatenate>
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv(".. /input/google-quest-challenge/test.csv")
 train = pd.read_csv(".. /input/google-quest-challenge/train.csv" )
train_smote = pd.concat([X_res, y_res], axis=1 )<merge>
question_output_columns = [col for col in output_columns \
if 'question' in col]
answer_output_colmns = [col for col in output_columns
if col not in question_output_columns]
train_smote_f= train_smote.merge(merged_drug_to_targets , on = 'drug_id' )<drop_column>
tokenizer = BertTokenizer.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-tokenizer' )
train_smote_f = train_smote_f.drop(['drug_id'], axis =1 )<prepare_x_and_y>
max_length_map = {'question_title': 32,
'question_body': 512,
'answer': 512
}
train_smote_f = train_smote_f.fillna(0)
 folds = train_smote_f.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train_smote_f.drop(target_cols, axis=1), y= train_smote_f[target_cols])) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<define_variables>
def txt_re(txt):
txt = txt.strip()
txt = re.sub('https?.*$', '', txt)
 txt = re.sub('https?.*\s', '', txt)
 txt = re.sub('
+', ' ', txt)
 txt = re.sub('\r+', ' ', txt)
 txt = re.sub('\t+', ' ', txt)
 txt = re.sub('&gt;', '>', txt)
 txt = re.sub('&lt;', '<', txt)
 txt = re.sub('&amp;', '&', txt)
 txt = re.sub('&quot;', '"', txt)
 return txt
train = train_smote_f<categorify>
def get_input(txt, pair_txt, tokenizer, max_length):
txt = txt_re(txt)
 txt = tokenizer.encode_plus(txt, pair_txt, add_special_tokens=True,max_length=max_length, \
pad_to_max_length='right')
 input_ids = txt['input_ids']
segment_masks = txt['token_type_ids']
input_masks = txt['attention_mask']
return input_ids, segment_masks, input_masks
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
question_train, answer_train = computer_input_array(train)
 question_test, answer_test = computer_input_array(test )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
labels = train[output_columns].values.tolist()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<categorify>
train_dict = {'question': question_train, 'answer': answer_train, 'label': labels}
test_dict = {'question': question_test, 'answer': answer_test}
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
torch.save(train_dict, './data/train_data.t7')
 torch.save(test_dict, './data/test_data.t7' )
feature_cols = [c for c in folds.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertTokenizer, BertModel
from scipy.stats import spearmanr
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 10
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500<normalization>
class Model_v1(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-pytorch-model')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output = nn.Linear(768 * 2, 30)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, \
a_inputs, a_input_masks, a_segment_masks):
q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze(1 ).squeeze(1)
 a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze(1 ).squeeze(1)
 t_q_a = torch.cat(( q_x, a_x), -1)
 output = self.output(t_q_a)
 x = torch.sigmoid(output)
 return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<prepare_x_and_y>
class Model_v2(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-pytorch-model')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output_q = nn.Linear(768, 21)
 self.output_a = nn.Linear(768, 9)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, \
a_inputs, a_input_masks, a_segment_masks):

 q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze(1 ).squeeze(1)
 a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze(1 ).squeeze(1)
 output_q = self.output_q(q_x)
 output_a = self.output_a(a_x)
 output = torch.cat(( output_q, output_a), -1)
 x = torch.sigmoid(output)
 return x
def run_training(fold, seed):
seed_everything(seed)
 train = folds
test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset, TensorDataset
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GroupKFold
from transformers import AdamW, get_linear_schedule_with_warmup
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
data = torch.load('./data/train_data.t7')
 test_data = torch.load('./data/test_data.t7' )
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<save_to_csv>
test_set = TensorDataset(torch.LongTensor(np.array(test_data['question'])) ,
torch.LongTensor(np.array(test_data['answer'])) )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<install_modules>
test_loader = DataLoader(
 test_set,
batch_size=1,
shuffle=False )
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
criterion = nn.BCELoss()
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"<load_from_csv>
models = []
for fold in range(5):
model_path = f'.. /input/google-qa-labeling-pretrained-v3/model_{fold}_2.t7'
if os.path.exists(model_path):
print(f'model available for prediction at {model_path}')
 model = Model_v1()
model.load_state_dict(torch.load(model_path))
models.append(model)
 for fold in range(5):
model_path = f'.. /input/google-qa-labeling-pretrained-v4/model_{fold}_2.t7'
if os.path.exists(model_path):
print(f'model available for prediction at {model_path}')
 model = Model_v2()
model.load_state_dict(torch.load(model_path))
models.append(model )
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 print(train.shape,test.shape)
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 print(train.shape,test.shape)
 train_col_size = train.shape[0]
cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 print(train.shape,test.shape)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
print(train.shape,test.shape)
 if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
print(train.shape,test.shape)
 if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
print(train.shape,test.shape)
 for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 print(train.shape,test.shape)
 features_to_drop = ["sig_id", "cp_type"]
train.drop(features_to_drop, axis = 1, inplace = True)
 test.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
print(train.shape,test.shape)
 train_df = train[: train_col_size]
train_df.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 X_test = test.values<load_from_csv>
final_predicts = []
for model in models:
model = model.cuda()
model.eval()
test_predicts = []
with torch.no_grad() :
for input_question, input_answer in test_loader:
print(input_answer.shape)
 input_question, input_answer = input_question.cuda() , input_answer.cuda()
input_question, input_answer = Variable(input_question, requires_grad=False), Variable(input_answer, requires_grad=False)
 scores = model(input_question[:,0],
input_question[:,2],
input_question[:,1],
input_answer[:,0],
input_answer[:,2],
input_answer[:,1])
 test_predicts.append(scores.reshape(scores.shape[-1]))
final_predicts.append(test_predicts )
<filter>
pres = np.average(final_predicts, axis=0)
 test_output = [[p.item() for p in pre] for pre in pres]
<categorify>
output_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
<categorify>
output_values = np.transpose(test_output ).tolist()
output_dict = {k: v for k, v in zip(output_cols, output_values)}
output_dict['qa_id'] = sample_submission['qa_id'].values.tolist()
output = pd.DataFrame.from_dict(output_dict )
<feature_engineering>
order = ['qa_id', 'question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
<choose_model_class>
output = output[order]
<import_modules><EOS> 
output.to_csv('submission.csv', index=False )