Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
def _get_masks1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids1(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input1(title, question, answer,tokenizer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs1(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids1(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks1(stoken, max_sequence_length)
 input_segments = _get_segments1(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays1(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input1(t, q, a,tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs1(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays1(df, columns):
return np.asarray(df[columns] )<string_transform>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(question,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 q_len = len(q)
 if(q_len+3)> max_sequence_length:
new_q_len = max_sequence_length-3
if new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_q_len+3))
q = q[:new_q_len]
return q
def _convert_to_bert_inputs(question, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_q(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_body
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_a(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.answer
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_t(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_title
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<find_best_params>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
coef_nn = 2
coef_mte = 1.7
coef_lgb =1<choose_model_class>
test_id = test['id']
del train['id']
del test['id']
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=2, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias<compute_test_metric>
encoder = OneHotEncoder(cols=['color'], use_cat_names=True)
 train = encoder.fit_transform(train)
 test = encoder.fit_transform(test )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 def average_spearmanr(y_true: np.ndarray, y_pred: np.ndarray)-> float:
return np.average([
spearmanr(y_t, y_p ).correlation for y_t, y_p in zip(y_true.T, y_pred.T)
 ])
 class SpearmanRhoCallback(Callback):
def __init__(self, training_data, validation_data, patience, model_name):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.patience = patience
self.value = -1
self.bad_epochs = 0
self.model_name = model_name
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_val = self.model.predict(self.x_val)
 rho_val = np.nanmean([spearmanr(self.y_val[:, ind], y_pred_val[:, ind] + np.random.normal(0, 1e-7, y_pred_val.shape[0])).correlation for ind in range(y_pred_val.shape[1])])
 if rho_val >= self.value:
self.value = rho_val
self.model.save_weights(self.model_name)
 else:
self.bad_epochs += 1
if self.bad_epochs >= self.patience:
print("Epoch %05d: early stopping Threshold" % epoch)
 self.model.stop_training = True
print('\rval_spearman-rho: %s' %(str(round(rho_val, 4))), end=100*' '+'
')
 return rho_val
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return<choose_model_class>
encoder = LabelEncoder()
encoder.fit(train['type'])
 print(encoder.classes_)
 train['type_no'] = encoder.transform(train['type'] )
class BertLayer1(tf.keras.layers.Layer):
def __init__(self, bert_path, seq_len=64, n_tune_layers=3,
pooling="cls", do_preprocessing=True, verbose=False,
tune_embeddings=False, trainable=True, **kwargs):
self.trainable = trainable
self.n_tune_layers = n_tune_layers
self.tune_embeddings = tune_embeddings
self.do_preprocessing = do_preprocessing
self.verbose = verbose
self.seq_len = seq_len
self.pooling = pooling
self.bert_path = bert_path
self.var_per_encoder = 16
if self.pooling not in ["cls", "mean", None]:
raise NameError(
 f"Undefined pooling type(must be either 'cls', 'mean', or None, but is {self.pooling}" 
)
 super(BertLayer1, self ).__init__(**kwargs)
 def build(self, input_shape):
self.bert = hub.Module(self.build_abspath(self.bert_path),
trainable=self.trainable, name=f"{self.name}_module")
 trainable_layers = []
if self.tune_embeddings:
trainable_layers.append("embeddings")
 if self.n_tune_layers > 0:
encoder_var_names = [var.name for var in self.bert.variables if 'encoder' in var.name]
n_encoder_layers = int(len(encoder_var_names)/ self.var_per_encoder)
 for i in range(self.n_tune_layers):
trainable_layers.append(f"encoder/layer_{str(n_encoder_layers - 1 - i)}/")
 for var in self.bert.variables:
if any([l in var.name for l in trainable_layers]):
self._trainable_weights.append(var)
 else:
self._non_trainable_weights.append(var)
 if self.verbose:
print("*** TRAINABLE VARS *** ")
 for var in self._trainable_weights:
print(var)
 self.build_preprocessor()
self.initialize_module()
super(BertLayer1, self ).build(input_shape)
 def build_abspath(self, path):
if path.startswith("https://")or path.startswith("gs://"):
return path
else:
return os.path.abspath(path)
 def build_preprocessor(self):
sess = tf.keras.backend.get_session()
tokenization_info = self.bert(signature="tokenization_info", as_dict=True)
 vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
tokenization_info["do_lower_case"]])
 self.preprocessor = build_preprocessor(vocab_file, self.seq_len, do_lower_case)
 def initialize_module(self):
sess = tf.keras.backend.get_session()
vars_initialized = sess.run([tf.is_variable_initialized(var)
 for var in self.bert.variables])
 uninitialized = []
for var, is_initialized in zip(self.bert.variables, vars_initialized):
if not is_initialized:
uninitialized.append(var)
 if len(uninitialized):
sess.run(tf.variables_initializer(uninitialized))
def call(self, input):
if self.do_preprocessing:
input = tf.numpy_function(self.preprocessor,
[input], [tf.int32, tf.int32, tf.int32],
name='preprocessor')
 for feature in input:
feature.set_shape(( None, self.seq_len))
input_ids, input_mask, segment_ids = input
bert_inputs = dict(
 input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids 
)
 output = self.bert(inputs=bert_inputs, signature="tokens", as_dict=True)
 pooled_cls = output["pooled_output"]
result = output["sequence_output"]
input_mask = tf.cast(input_mask, tf.float32)
 mul_mask = lambda x, m: x * tf.expand_dims(m, axis=-1)
 masked_reduce_mean = lambda x, m: tf.reduce_sum(mul_mask(x, m), axis=1)/(
 tf.reduce_sum(m, axis=1, keepdims=True)+ 1e-10)
 pooled_mean = masked_reduce_mean(result, input_mask)
 sequence_output = mul_mask(result, input_mask)
 if self.pooling == "cls":
return pooled_cls
else:
return pooled_mean, sequence_output
def get_config(self):
config_dict = {
"bert_path": self.bert_path,
"seq_len": self.seq_len,
"pooling": self.pooling,
"n_tune_layers": self.n_tune_layers,
"tune_embeddings": self.tune_embeddings,
"do_preprocessing": self.do_preprocessing,
"verbose": self.verbose
}
super(BertLayer1, self ).get_config()
return config_dict<categorify>
target = train['type_no']
target_string = train['type']
del train['type']
del train['type_no']
target.head()
outputs = compute_output_arrays1(df_train, output_categories)
 test_inputs = compute_input_arays1(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputst = compute_input_arays_t(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_t)
 test_inputsq = compute_input_arays_q(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_q)
 test_inputsa = compute_input_arays_a(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_a )<categorify>
train_data, test_data, train_target, test_target = train_test_split(train, target, test_size=0.2, random_state=42 )
d= {0.: 0, 0.3: 1, 0.7: 2}
theArray = np.round(outputs[:, 19:20], decimals=1)
 newArray = copy(theArray)
 for k, v in d.items() : newArray[theArray==k] = v
newArray = np.squeeze(newArray)
 newArray =newArray.astype(int)
 np.unique(newArray )<drop_column>
decisions(DecisionTreeClassifier() , ['hair_length', 'bone_length'] )
del test_inputs, test_inputst, test_inputsq,test_inputsa; gc.collect()<categorify>
Image(url= 'https://cdn-images-1.medium.com/max/1600/1*JZbxrdzabrT33Yl-LrmShw.png', width=750, height=750 )
def _get_masks2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids2(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input2(question, answer,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 q_len = len(q)
 a_len = len(a)
 if(q_len+a_len+3)> max_sequence_length:
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(ceil(new_q_len)) , int(floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
q = q[:new_q_len]
a = a[:new_a_len]
return q, a
def _convert_to_bert_inputs2(question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids2(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks2(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays2(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q, a = instance.question_title, instance.question_body
q, a = _trim_input2(q, a, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs2(
 q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]<categorify>
clfs = [RandomForestClassifier() , AdaBoostClassifier() , SVC() , KNeighborsClassifier() ]
labels = ['Random Forest', 'Ada Boost', 'Support Vector', 'K-Neighbors']
for model, name in zip(clfs, labels):
model.fit(train_data, train_target)
 predictions = model.predict(test_data)
 print('{} accuracy is: {}'.format(name, accuracy_score(test_target, predictions)) )
def _get_masks3(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments3(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids3(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input3(question, answer,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 q_len = len(q)
 a_len = len(a)
 if(q_len+a_len+3)> max_sequence_length:
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(ceil(new_q_len)) , int(floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
q = q[:new_q_len]
a = a[:new_a_len]
return q, a
def _convert_to_bert_inputs3(question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids3(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks3(stoken, max_sequence_length)
 input_segments = _get_segments3(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays3(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q, a = instance.question_title, instance.answer
q, a = _trim_input3(q, a,tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs3(
 q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)] 
<categorify>
params = {
'C': [0.001, 0.01, 0.1, 1, 10],
'gamma': [0.001, 0.01, 0.1, 1]
}
grid_search = GridSearchCV(SVC() , params, cv=5)
 grid_search.fit(train, target)
 grid_search.best_params_
def _get_masks1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids1(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input1(title, question, answer,tokenizer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs1(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids1(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks1(stoken, max_sequence_length)
 input_segments = _get_segments1(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays1(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input1(t, q, a,tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs1(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays1(df, columns):
return np.asarray(df[columns] )<string_transform>
importance_model = SVC(C=10, gamma=0.1, probability=True)
 importance_model.fit(train, target)
 perm = PermutationImportance(importance_model, random_state=42 ).fit(test_data, test_target)
 eli5.show_weights(perm, feature_names=test_data.columns.tolist() )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(question,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 q_len = len(q)
 if(q_len+3)> max_sequence_length:
new_q_len = max_sequence_length-3
if new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_q_len+3))
q = q[:new_q_len]
return q
def _convert_to_bert_inputs(question, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_q(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_body
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_a(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.answer
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_t(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_title
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<statistical_test>
test_target.iloc[0], target_string.iloc[0]
outputs = compute_output_arrays1(df_train, output_categories)
 test_inputs = compute_input_arays1(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputstq = compute_input_arays2(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 inputsq = compute_input_arays_q(df_train, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_q)
 test_inputst = compute_input_arays_t(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_t)
 test_inputsq = compute_input_arays_q(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_q)
 test_inputsa = compute_input_arays_a(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_a )<load_pretrained>
model = SVC(C=10, gamma=0.1)
 model.fit(train, target_string )
%%time
kf = MultilabelStratifiedKFold(n_splits=10, random_state=42, shuffle=True)
 for i,(tr, val)in enumerate(kf.split(inputsq[1], outputs)) :
if i!=7:
continue
K.clear_session()
print('starting fold: {0}'.format(i))
json_file = open('.. /input/save-groupbertq-fold{}/nn_modelq{}.json'.format(i, i), 'r')
 loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json, custom_objects={"BertLayer1": BertLayer1})
 bst_model_path = '.. /input/save-groupbertq-fold{}/dl_modelq{}.h5'.format(i, i)
 print(bst_model_path)
 model.load_weights(bst_model_path)
 model.compile(optimizer=tf.keras.optimizers.Nadam(3e-5),loss=['binary_crossentropy'])
 print("Loaded model from disk")
 y_testq = model.predict([test_inputstq[0],test_inputstq[1],test_inputstq[2],test_inputst[0],test_inputst[1],test_inputst[2],
test_inputsq[0],test_inputsq[1],test_inputsq[2]])
 print("y_testq shape", y_testq.shape)
 del model
gc.collect()
K.clear_session()
print('starting fold: {0}'.format(i))
json_file = open('.. /input/save-groupberta-fold{}/nn_modela{}.json'.format(i, i), 'r')
 loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json, custom_objects={"BertLayer1": BertLayer1})
 bst_model_path = '.. /input/save-groupberta-fold{}/dl_modela{}.h5'.format(i, i)
 print(bst_model_path)
 model.load_weights(bst_model_path)
 model.compile(optimizer=tf.keras.optimizers.Nadam(3e-5),loss=['binary_crossentropy'])
 print("Loaded model from disk")
 y_testa = model.predict([test_inputs[0],test_inputs[1],test_inputs[2],test_inputst[0],test_inputst[1],test_inputst[2],
test_inputsq[0],test_inputsq[1],test_inputsq[2],test_inputsa[0],test_inputsa[1],test_inputsa[2]])
 print("y_testa shape", y_testa.shape)
 print("Saved model to disk")
 del model
gc.collect()
y_test = np.concatenate(( y_testq, y_testa), axis=1)
 print("y_test shape", y_test.shape)
 all_predictions.append(y_test)
 gc.collect() 
<prepare_output>
predictions = model.predict(test)
 predictions[:10]
test_preds = np.mean(all_predictions, axis=0 )<define_variables>
submission = pd.DataFrame({'id': test_id, 'type': predictions})
 submission.head()
output =0.23*output_roberta + 0.31*output_xlnet + 0.24*test_preds + 0.22*final_predictions<define_variables>
submission.to_csv('submission.csv', index=False )
culture_category_list = list(df_test[(df_test['host'] == 'english.stackexchange.com')|(df_test['host'] == 'ell.stackexchange.com')].index )<define_variables>
submission.to_csv('submission.csv', index=False )