Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<load_pretrained>

 N_SPLITS = 5
BATCH_SIZE = 128
EPOCHS = 50
HID = 64
LR = 0.001
SIGNAL_TYPE = 'denoised'
FEATURE_TIME_STEPS = 160
test_predictions=[]
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold<5:
keras.backend.clear_session()
model = bertModel()
print(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_fold_{fold}.hdf5')
 model.load_weights(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_fold_{fold}.hdf5')
 preds = model.predict(test_inputs_q +
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values]+test_inputs_a,
batch_size=8, verbose=1)
 test_predictions.append(preds)
 print(f'Fold:{fold} Completed')
 if fold >=7:
keras.backend.clear_session()
model = bertModel()
print(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_fold_{fold}.hdf5')
 model.load_weights(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_fold_{fold}.hdf5')
 preds = model.predict(test_inputs_q +
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values]+test_inputs_a,
batch_size=8, verbose=1)
 test_predictions.append(preds)
 print(f'Fold:{fold} Completed' )<normalization>
def _get_masks_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_google_qa(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input_google_qa(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer_google_qa.tokenize(title)
 q = tokenizer_google_qa.tokenize(question)
 a = tokenizer_google_qa.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_google_qa(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_google_qa(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks_google_qa(stoken, max_sequence_length)
 input_segments = _get_segments_google_qa(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_google_qa(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance['treated_question_title'], instance['treated_question_body'], instance['treated_answer']
t, q, a = _trim_input_google_qa(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_google_qa(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays_google_qa(df, columns):
return np.asarray(df[columns])
 def bert_model_google_qa(learning_rate=4e-5,loss_function='binary_crossentropy'):
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[[input_word_ids, input_masks, input_segments]], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 return model 
<split>
sample_size = 800000
gkf_google_qa = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs_google_qa = compute_output_arrays_google_qa(df_train, output_categories)
 inputs_google_qa = compute_input_arays_google_qa(df_train, ['treated_question_title','treated_question_body','treated_answer'], tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
 test_inputs_google_qa = compute_input_arays_google_qa(df_test, ['treated_question_title','treated_question_body','treated_answer'],tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
<choose_model_class>
def matthews_correlation(y_true, y_pred):
y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
test_predictions_google_qa=[]
for fold,(train_idx, valid_idx)in enumerate(gkf_google_qa):
if fold<7:
keras.backend.clear_session()
model_qa = bert_model_google_qa()
print(f'/kaggle/input/bertbase/bert-base-{fold}-4.h5py')
 model_qa.load_weights(f'/kaggle/input/bertbase/bert-base-{fold}-4.h5py')
 preds_qa = model_qa.predict(test_inputs_google_qa , batch_size=8, verbose=1)
 test_predictions_google_qa.append(preds_qa)
 print(f'Fold:{fold} Completed')
 if fold < 10 and fold >=7:
keras.backend.clear_session()
model_qa = bert_model_google_qa()
print(f'/kaggle/input/bertbaseextra/bert-base-{fold}-4.h5py')
 model_qa.load_weights(f'/kaggle/input/bertbaseextra/bert-base-{fold}-4.h5py')
 preds_qa = model_qa.predict(test_inputs_google_qa , batch_size=8, verbose=1)
 test_predictions_google_qa.append(preds_qa)
 print(f'Fold:{fold} Completed' )<normalization>

 def my_mat_cor(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1))
tn = np.sum(( y_true == 0)&(y_pred == 0))
fp = np.sum(( y_true == 0)&(y_pred == 1))
fn = np.sum(( y_true == 1)&(y_pred == 0))
numerator =(tp * tn - fp * fn)
 denominator =(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn)) **.5
return numerator /(denominator + 1e-15)
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape<compute_test_metric>
df_train = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head(9 )
final_preds = np.average(np.array([test_preds, test_preds_google_qa]),axis=0)
 final_preds.shape<save_to_csv>
!ls.. /input/vsb-wavelet-denoised
parq_sample_dn = pq.read_pandas('.. /input/vsb-wavelet-denoised/train_dn.parquet', columns=[str(i)for i in range(0, 6)] ).to_pandas()
parq_sample = pq.read_pandas('.. /input/vsb-power-line-fault-detection/train.parquet', columns=[str(i)for i in range(0, 6)] ).to_pandas()
df_sub.iloc[:, 1:] = final_preds
df_sub.to_csv('submission.csv', index=False )<import_modules>
del parq_sample_dn, parq_sample
pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )<load_from_csv>
max_num = 127
min_num = -128
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )<load_from_csv>
def transform_ts(ts, n_dim=FEATURE_TIME_STEPS, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )<load_from_csv>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/vsb-wavelet-denoised/train_dn.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )<define_variables>
X_orig = np.load('.. /input/5-fold-lstm-attention-fully-commented-0-694/X.npy')
 print(X_orig.shape )
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
] 
<feature_engineering>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x0 = Bidirectional(CuDNNLSTM(HID, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(HID, return_sequences=True))(x0)
 x1 = GlobalAveragePooling1D()(x)
 x2 = GlobalMaxPooling1D()(x)
 x = Concatenate()([x1,x2])
 x = Dense(64, activation="relu" )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(lr=LR), metrics=[matthews_correlation])
 return model
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )<count_unique_values>
splits = list(StratifiedKFold(n_splits=N_SPLITS ).split(X, y))
preds_val_list = []
y_val_list = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 csv_logger = CSVLogger('trainlog.csv', append=True, separator=',')
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx),
save_best_only=True, save_weights_only=True,
verbose=1, monitor='val_matthews_correlation',
mode='max')
 model.fit(train_X, train_y, batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=[val_X, val_y],
callbacks=[ckpt,csv_logger], verbose=0)
 model.load_weights('weights_{}.h5'.format(idx))
preds_1fold = model.predict(val_X, batch_size=512)
 result = threshold_search(val_y, preds_1fold[:,0])
 print('optimized threshold is %.2f and have score %.4f' % 
(result['threshold'], result['matthews_correlation']))
preds_val_list.append(preds_1fold)
 y_val_list.append(val_y)
 preds_val = np.concatenate(preds_val_list)[...,0]
y_val = np.concatenate(y_val_list)
 print(preds_val.shape, y_val.shape)
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value 
<set_options>
%%time
meta_test = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_test.csv' )
np.set_printoptions(suppress=True )<load_from_csv>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
X_test_dn = np.asarray([np.concatenate([X_test_raw[i][3],X_test_raw[i+1][3], X_test_raw[i+2][3]], axis=1)for i in range(0,len(X_test_raw), 3)])
 np.save("X_test_dn.npy",X_test_dn)
 X_test_dn.shape
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
X_test_orig = np.load('.. /input/5-fold-lstm-attention-fully-commented-0-694/X_test.npy')
 print(X_test_orig.shape )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<split>
result = threshold_search(y_val, preds_val)
 best_threshold = result['threshold']
print(result['threshold'],result['matthews_correlation'] )
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<load_pretrained>
preds_test_prob = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test, batch_size=300, verbose=1)
 pred_3 = np.repeat(pred,3)
 preds_test_prob.append(pred_3)
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
<load_pretrained>
preds_test_prob = np.squeeze(np.mean(preds_test_prob, axis=0))
np.save('preds_test_prob.npy',preds_test_prob)
 print(preds_test_prob.shape )
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
preds_test =(preds_test_prob > best_threshold ).astype(np.int)
 print(preds_test.shape)
 print(sum(preds_test))
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
submission = pd.read_csv('.. /input/vsb-power-line-fault-detection/sample_submission.csv')
 print(len(submission))
submission.head()
for i in range(7):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()
for i in range(7):
model_path = f'.. /input/bert-base-tf2-0-training/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<define_variables>
submission25 = submission.copy()
submission30 = submission.copy()
submission40 = submission.copy()
submission55 = submission.copy()
submission60 = submission.copy()
<predict_on_test><EOS> 
preds_test25 =(preds_test_prob > 0.25 ).astype(int)
 preds_test30 =(preds_test_prob > 0.30 ).astype(int)
 preds_test40 =(preds_test_prob > 0.40 ).astype(int)
 preds_test55 =(preds_test_prob > 0.55 ).astype(int)
 preds_test60 =(preds_test_prob > 0.60 ).astype(int)
 submission25['target'] = preds_test25
submission30['target'] = preds_test30
submission40['target'] = preds_test40
submission55['target'] = preds_test55
submission60['target'] = preds_test60
submission25.to_csv('submission25.csv', index=False)
 submission30.to_csv('submission30.csv', index=False)
 submission40.to_csv('submission40.csv', index=False)
 submission55.to_csv('submission55.csv', index=False)
 submission60.to_csv('submission60.csv', index=False )