Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<choose_model_class>
BS = 8
SEED = 42
def train_dnn_classifier_text(learning_rate=.05, steps=2000, batch_size=100, hidden_units=[20,20], dropout=.1):
my_optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
 my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
 feature_columns = construct_feature_columns()
classifier = tf.estimator.DNNClassifier(
 feature_columns=feature_columns,
hidden_units=hidden_units,
optimizer=my_optimizer,
dropout=dropout 
)
 classifier.train(
 input_fn=lambda: tfrecord_input_fn(TRAINING_TFRECORD, batch_size=batch_size),
steps=steps)
 predict_training_input_fn = lambda: tfrecord_input_fn(TRAINING_TFRECORD,
batch_size=batch_size,
num_epochs=1,
shuffle=False)
 predict_validation_input_fn = lambda: tfrecord_input_fn(VALIDATION_TFRECORD,
batch_size=batch_size,
num_epochs=1,
shuffle=False)
 return classifier, [predict_training_input_fn, predict_validation_input_fn] 
<train_model>
!pip install.. /input/sacremoses/sacremoses-master/ > /dev/null
!pip install.. /input/transformers/transformers-master/ > /dev/null
if(USING_OLD_MODELS == False)&(IS_DNN_CLASSIFIER):
dnn_classifier, dnn_arr_funcs = train_dnn_classifier_text(learning_rate=.005,
steps=1500,
batch_size=100,
hidden_units=[6],
dropout=.1 )<predict_on_test>
tqdm.pandas()
device = torch.device('cuda' )
predict_test_input_fn = lambda: tfrecord_input_fn(TEST_TFRECORD,
num_epochs=1,
shuffle=False)
 if(USING_OLD_MODELS == False):
if(IS_DNN_CLASSIFIER == False):
predictions_generator = classifier.predict(input_fn=predict_test_input_fn)
 else:
predictions_generator = dnn_classifier.predict(input_fn=predict_test_input_fn)
 predictions_list = list(predictions_generator)
 probabilities = [p["probabilities"][1] for p in predictions_list]
print('Now have the probabilities.' )<create_dataframe>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv' ).fillna(' ')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv' ).fillna(' ')
 sub = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv' ).fillna(' ' )
submission_values = pd.DataFrame({'id': combined_test_dataset['id'], 'project_is_approved': probabilities})
 submission_values[0:5]<save_to_csv>
def _get_segments_xlnet(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "<sep>":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return [0] *(max_seq_length - len(tokens)) + segments
def _get_segments_bert(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_xlnet(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = [5] *(max_seq_length-len(token_ids)) + token_ids
return input_ids
def _get_ids_bert(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length=512-1,
t_max_len=70-1, q_max_len=219, a_max_len=219):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = norm_token_length(q, q_new_len)
 a = norm_token_length(a, a_new_len)
 return t, q, a
def norm_token_length(tokens, l):
if len(tokens)> l:
head = l//2
tail = l - head
return tokens[:head] + tokens[-tail:]
else:
return tokens[:l]
def _convert_to_bert_inputs(title, question, answer, cate, pretrained_weights, max_sequence_length=512):

 if "bert-base" in pretrained_weights:
stoken = ["[CLS]"] + [cate] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_bert(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_bert(stoken, max_sequence_length)
 elif pretrained_weights == "xlnet-base-cased":
stoken = [cate] + title + ["<sep>"] + question + ["<sep>"] + answer + ["<sep>", "<cls>"]
input_ids = _get_ids_xlnet(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_xlnet(stoken, max_sequence_length)
 try:
cls_index = input_segments.index(5)- 1
except ValueError:
cls_index = -1
input_segments[cls_index] = 2
return [input_ids, input_segments]
def convert_row(row, pretrained_weights):
if pretrained_weights == "bert-base-uncased":
c = f"[{row['category'].lower() }]"
elif pretrained_weights == "bert-base-cased":
c = f"[{row['category']}]"
elif pretrained_weights == "xlnet-base-cased":
c = f"[{row['category']}]"
t, q, a = title = row["question_title"], row["question_body"], row["answer"]
t, q, a = _trim_input(t, q, a)
 ids, segments = _convert_to_bert_inputs(t, q, a, c, pretrained_weights)
 return np.array([[ids, segments]] )
submission_values.to_csv('linear-spelling.csv', index=False )<define_variables>
class CustomBert(BertPreTrainedModel):
def __init__(self, config):
super(CustomBert, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.classifier = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 pooled_output = torch.cat([outputs[2][-1*i][:,0] for i in range(1, self.n_use_layer+1)], dim=1)
 pooled_output = self.dense1(pooled_output)
 pooled_output = self.dense2(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
COMPLETE_RUN = False<set_options>
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bert-swa/')
 tokenizer.added_tokens_encoder
os.environ['PYTHONHASHSEED'] = '10000'
np.random.seed(10001)
 random.seed(10002)
 tf.set_random_seed(10003)
 session_conf = tf.ConfigProto(intra_op_parallelism_threads=6, inter_op_parallelism_threads=5)
 backend.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))<define_variables>
pretrained_weights = 'bert-base-uncased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )
max_size = 150000
max_features = 100000
maxlen = 500
embed_size = 300
embedding_file = '.. /input/donorschooseorg-preprocessed-data/embeddings_small.vec'
if COMPLETE_RUN == False:
max_size = 150
max_features = 100
maxlen = 5
embedding_file = '.. /input/donorschooseorg-preprocessed-data/embeddings_small.vec'<load_from_csv>
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()
base = ".. /input/donorschoose-application-screening/"
train = pd.read_csv(base + "train.csv")
 test = pd.read_csv(base + "test.csv")
 resources = pd.read_csv(base + "resources.csv")
 train = train.sort_values(by="project_submitted_datetime" )<split>
model_dir_target = ".. /input/quest-bertuncased-10fold/"
bert_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-uncased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 bert_pred_lst.append(test_pred )
if COMPLETE_RUN == False:
train = train[:200]
test = test[:200]
resources = resources[:100]<define_variables>
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bertcased-tokenizer/')
 tokenizer.added_tokens_encoder
char_cols = ['project_title', 'project_essay_1', 'project_essay_2',
'project_essay_3', 'project_essay_4', 'project_resource_summary']<create_dataframe>
pretrained_weights = 'bert-base-cased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )
resources['total_price'] = resources.quantity * resources.price
mean_total_price = pd.DataFrame(resources.groupby('id' ).total_price.mean())
 sum_total_price = pd.DataFrame(resources.groupby('id' ).total_price.sum())
 count_total_price = pd.DataFrame(resources.groupby('id' ).total_price.count())
 mean_total_price['id'] = mean_total_price.index
sum_total_price['id'] = mean_total_price.index
count_total_price['id'] = mean_total_price.index<merge>
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()
def create_features(df):
df = pd.merge(df, mean_total_price, on='id')
 df = pd.merge(df, sum_total_price, on='id')
 df = pd.merge(df, count_total_price, on='id')
 df['year'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[0])
 df['month'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[1])
 for col in char_cols:
df[col] = df[col].fillna("NA")
 df['text'] = df.apply(lambda x: " ".join(x[col] for col in char_cols), axis=1)
 return df<feature_engineering>
model_dir_target = ".. /input/quest-bertcased-10fold/"
cased_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 cased_pred_lst.append(test_pred )
train = create_features(train)
 test = create_features(test )<define_variables>
class CustomXLNet(XLNetForSequenceClassification):
def __init__(self, config):
super(CustomXLNet, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.n_labels = config.num_labels
self.transformer = XLNetModel(config)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.logits_proj = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(
 self,
input_ids=None,
attention_mask=None,
mems=None,
perm_mask=None,
target_mapping=None,
token_type_ids=None,
input_mask=None,
head_mask=None,
inputs_embeds=None,
labels=None, 
):
transformer_outputs = self.transformer(
 input_ids,
attention_mask=attention_mask,
mems=mems,
perm_mask=perm_mask,
target_mapping=target_mapping,
token_type_ids=token_type_ids,
input_mask=input_mask,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 output = torch.cat([transformer_outputs[1][-1*(i+1)][:,-1] for i in range(self.n_use_layer)], dim=1)
 output = self.dense1(output)
 output = self.dense2(output)
 logits = self.logits_proj(output)
 outputs =(logits,)+ transformer_outputs[1:]
return outputs
cat_features = ["teacher_prefix", "school_state", "year", "month", "project_grade_category",
"project_subject_categories", "project_subject_subcategories"]
num_features = ["teacher_number_of_previously_posted_projects", "total_price_x",
"total_price_y", "total_price"]
cat_features_hash = [col+"_hash" for col in cat_features]<feature_engineering>
tokenizer = XLNetTokenizer.from_pretrained('.. /input/quest-xlnet-tokenizer/')
 tokenizer.added_tokens_encoder
def feature_hash(df, max_size=max_size):
for col in cat_features:
df[col+"_hash"] = df[col].apply(lambda x: hash(x)%max_size)
 return df
train = feature_hash(train)
 test = feature_hash(test )<prepare_x_and_y>
pretrained_weights = "xlnet-base-cased"
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )
scaler = StandardScaler()
X_train_num = scaler.fit_transform(train[num_features])
 X_test_num = scaler.transform(test[num_features])
 X_train_cat = np.array(train[cat_features_hash], dtype=np.int)
 X_test_cat = np.array(test[cat_features_hash], dtype=np.int)
 tokenizer = text.Tokenizer(num_words=max_features )<categorify>
model = CustomXLNet.from_pretrained(".. /input/quest-xlnet-tokenizer/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()
def preprocess(string):

 string = re.sub(r'(")', ' ', string)
 string = re.sub(r'(\r)', ' ', string)
 string = re.sub(r'(
)', ' ', string)
 string = re.sub(r'(\r 
)', ' ', string)
 string = re.sub(r'(\\)', ' ', string)
 string = re.sub(r'\t', ' ', string)
 string = re.sub(r'\:', ' ', string)
 string = re.sub(r'<string_transform>
model_dir_target = ".. /input/quest-xlnetcased-10fold/"
xlnet_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/xlnet-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids != 5 ).int().to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 xlnet_pred_lst.append(test_pred )
tokenizer.fit_on_texts(train["text"].tolist() +test["text"].tolist())
 list_tokenized_train = tokenizer.texts_to_sequences(train["text"].tolist())
 list_tokenized_test = tokenizer.texts_to_sequences(test["text"].tolist())
 X_train_words = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_test_words = sequence.pad_sequences(list_tokenized_test, maxlen=maxlen)
 X_train_target = train.project_is_approved<categorify>
def sigmoid(x):
return 1.0 /(1.0 + np.exp(-x))
bert_pred = np.array(bert_pred_lst ).mean(0)
 cased_pred = np.array(cased_pred_lst ).mean(0)
 xlnet_pred = np.array(xlnet_pred_lst ).mean(0)
 bert_pred = sigmoid(bert_pred)
 cased_pred = sigmoid(cased_pred)
 xlnet_pred = sigmoid(xlnet_pred)
 test_pred = bert_pred*0.4 + cased_pred*0.2 + xlnet_pred*0.4
embeddings_index = {}
with open(embedding_file, encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs<feature_engineering>
lgbm_models = pickle.load(open(".. /input/quest-lgbm/lgbm_question_type_spelling.pkl", 'rb'))
count_vectorizers = pickle.load(open(".. /input/quest-lgbm/tfidf_vectorizers.pkl", 'rb'))
word_index = tokenizer.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<import_modules>
dfs = []
for idx, col_name in enumerate(["question_title", "question_body", "answer"]):
X = count_vectorizers[idx].transform(test[col_name])
 feat = [f"{col_name}_{c}".encode("utf-8")for c in count_vectorizers[idx].get_feature_names() ]
df = pd.DataFrame(X.toarray() , columns=feat)
 dfs.append(df)
 test_x = pd.concat(dfs, axis=1)
 test_x.shape
GlobalMaxPool1D, GlobalAveragePooling1D, SpatialDropout1D, CuDNNGRU, Bidirectional, GRU, BatchNormalization 
<define_search_space>
lgbm_pred = np.zeros(len(test_x))
for fold in range(4):
lgbm_pred += lgbm_models[fold].predict(test_x)/4
lgbm_pred.shape
recurrent_units = 96
convolution_filters = 128
dense_units = [256, 128, 64]
dropout_rate = 0.3
learning_rate = 5e-3
if COMPLETE_RUN == False:
recurrent_units = 8
convolution_filters = 8
dense_units = [8, 4, 4]<choose_model_class>
def spearman_corr(y_true, y_pred):
if np.ndim(y_pred)== 2:
corr = np.nan_to_num([stats.spearmanr(y_true[:, i], y_pred[:, i])[0] for i in range(y_true.shape[1])] ).mean()
else:
corr = stats.spearmanr(y_true, y_pred)[0]
return corr
class OptimizedRounder(object):
def __init__(self,
n_overall: int = 5,
n_classwise: int = 5,
n_classes: int = 7,
metric: str = "qwk"):
self.n_overall = n_overall
self.n_classwise = n_classwise
self.n_classes = n_classes
self.coef = [1.0 / n_classes * i for i in range(1, n_classes)]
self.metric_str = metric
self.metric = spearman_corr
def _loss(self, X: np.ndarray, y: np.ndarray)-> float:
X_p = np.digitize(X, self.coef)
 ll = -self.metric(y, X_p)
 return ll
def fit(self, X: np.ndarray, y: np.ndarray):
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [ 
(0.01, 1.0 / self.n_classes + 0.05),
]
for i in range(1, self.n_classes):
ab_start.append(( i * 1.0 / self.n_classes + 0.05, 
(i + 1)* 1.0 / self.n_classes + 0.05))
for _ in range(self.n_overall):
for idx in range(self.n_classes - 1):
a, b = ab_start[idx]
self.coef[idx] = a
la = self._loss(X, y)
 self.coef[idx] = b
lb = self._loss(X, y)
 for it in range(self.n_classwise):
if la > lb:
a = b -(b - a)* golden1
self.coef[idx] = a
la = self._loss(X, y)
 else:
b = b -(b - a)* golden2
self.coef[idx] = b
lb = self._loss(X, y)
 def predict(self, X: np.ndarray)-> np.ndarray:
X_p = np.digitize(X, self.coef)
 return X_p
def get_model() :
input_cat = Input(( len(cat_features_hash),))
input_num = Input(( len(num_features),))
input_words = Input(( maxlen,))
x_cat = Embedding(max_size, 20 )(input_cat)
 x_cat = SpatialDropout1D(dropout_rate )(x_cat)
 x_cat = Flatten()(x_cat)
 x_words = Embedding(max_features, 300,
weights=[embedding_matrix],
trainable=False )(input_words)
 x_words = SpatialDropout1D(dropout_rate )(x_words)
 x_words1 = Bidirectional(GRU(recurrent_units, return_sequences=True))(x_words)
 x_words1 = Convolution1D(convolution_filters, 3, activation="relu" )(x_words1)
 x_words1_1 = GlobalMaxPool1D()(x_words1)
 x_words1_2 = GlobalAveragePooling1D()(x_words1)
 x_words2 = Convolution1D(convolution_filters, 2, activation="relu" )(x_words)
 x_words2 = Convolution1D(convolution_filters, 2, activation="relu" )(x_words2)
 x_words2_1 = GlobalMaxPool1D()(x_words2)
 x_words2_2 = GlobalAveragePooling1D()(x_words2)
 x_num = input_num
x = concatenate([x_words1_1, x_words1_2, x_words2_1, x_words2_2, x_cat, x_num])
 x = BatchNormalization()(x)
 x = Dense(dense_units[0], activation="relu" )(x)
 x = Dense(dense_units[1], activation="relu" )(x)
 x = concatenate([x, x_num])
 x = Dense(dense_units[2], activation="relu" )(x)
 predictions = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_cat, input_num, input_words], outputs=predictions)
 model.compile(optimizer=optimizers.Adam(learning_rate, decay=1e-6),
loss='binary_crossentropy',
metrics=['accuracy'])
 return model<find_best_params>
optR_lst = pickle.load(open(".. /input/quest-optimizedrounder/optR_lst_10fold_ensemble_v3.pkl", 'rb'))
lst = []
for idx, optR in enumerate(optR_lst):
coeff = optR.predict(test_pred[:, idx])
 lst.append(coeff/norm_dict[sub.columns[1:][idx]])
 opt_preds = np.array(lst ).T
test_pred = opt_preds
test_pred[:,19] = lgbm_pred
model = get_model()<define_variables>
test_pred[test[test["category"] != "CULTURE"].index, 19] = 0.0
test["host_info"] = test["question_user_page"].map(lambda x: x.split("/")[2].replace(".stackexchange.com", ""))
test_pred[test[test["host_info"].map(lambda x: x not in ["english", "ell"])].index, 19] = 0.0
<train_model><EOS> 
sub[sub.columns[1:]] = test_pred
sub.to_csv("submission.csv", index=False)
 sub.head()