Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<count_values>
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import *
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
from keras import activations
from keras import regularizers
from keras import initializers
from keras import constraints
from sklearn.preprocessing import MinMaxScaler
from numba import jit
from math import log, floor
from sklearn.neighbors import KDTree
from scipy.signal import periodogram, welch
from keras.engine import Layer
from keras.engine import InputSpec
from keras.objectives import categorical_crossentropy
from keras.objectives import sparse_categorical_crossentropy
train_category = train['category'].value_counts()
test_category = test['category'].value_counts()<count_unique_values>
N_SPLITS = 5
sample_size = 800000
train_question_user = train['question_user_name'].unique()
test_question_user = test['question_user_name'].unique()
print('Number of unique question user in train: ', len(train_question_user))
print('Number of unique question user in test : ', len(test_question_user))
print('Number of unique question user in both train & test : ', len(set(train_question_user)& set(test_question_user)) )<count_unique_values>
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
train_answer_user = train['answer_user_name'].unique()
test_answer_user = test['answer_user_name'].unique()
print('Number of unique answer user in train: ', len(train_answer_user))
print('Number of unique answer user in test : ', len(test_answer_user))
print('Number of unique answer user in both train & test : ', len(set(train_answer_user)& set(test_answer_user)) )<count_unique_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
print('Number of unique user in both question & answer in train : ', len(set(train_answer_user)& set(train_question_user)))
 print('Number of unique user in both question & answer in test : ', len(set(test_answer_user)& set(test_question_user)) )<count_values>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
def char_count(s):
return len(s)
 def word_count(s):
return s.count(' ' )<feature_engineering>
max_num = 127
min_num = -128
train['question_title_n_chars'] = train['question_title'].apply(char_count)
 train['question_title_n_words'] = train['question_title'].apply(word_count)
 train['question_body_n_chars'] = train['question_body'].apply(char_count)
 train['question_body_n_words'] = train['question_body'].apply(word_count)
 train['answer_n_chars'] = train['answer'].apply(char_count)
 train['answer_n_words'] = train['answer'].apply(word_count)
 test['question_title_n_chars'] = test['question_title'].apply(char_count)
 test['question_title_n_words'] = test['question_title'].apply(word_count)
 test['question_body_n_chars'] = test['question_body'].apply(char_count)
 test['question_body_n_words'] = test['question_body'].apply(word_count)
 test['answer_n_chars'] = test['answer'].apply(char_count)
 test['answer_n_words'] = test['answer'].apply(word_count )<create_dataframe>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
def calc_corr(df, x_cols, y_cols):
arr1 = df[x_cols].T.values
arr2 = df[y_cols].T.values
corr_df = pd.DataFrame(1 - cdist(arr2, arr1, metric='correlation'), index=y_cols, columns=x_cols)
 return corr_df<compute_test_metric>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
number_feature_cols = ['question_title_n_chars', 'question_title_n_words', 'question_body_n_chars', 'question_body_n_words', 'answer_n_chars', 'answer_n_words']
corr_df = calc_corr(train, target_cols, number_feature_cols )<count_values>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
num_question = train['question_user_name'].value_counts()
num_answer = train['answer_user_name'].value_counts()<categorify>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
train['num_answer_user'] = train['answer_user_name'].map(num_answer)
 train['num_question_user'] = train['question_user_name'].map(num_question)
 test['num_answer_user'] = test['answer_user_name'].map(num_answer)
 test['num_question_user'] = test['question_user_name'].map(num_question )<compute_test_metric>
def petrosian_fd(x):

 n = len(x)
 diff = np.ediff1d(x)
 N_delta =(diff[1:-1] * diff[0:-2] < 0 ).sum()
return np.log10(n)/(np.log10(n)+ np.log10(n /(n + 0.4 * N_delta)))
 def katz_fd(x):

 x = np.array(x)
 dists = np.abs(np.ediff1d(x))
ll = dists.sum()
ln = np.log10(np.divide(ll, dists.mean()))
aux_d = x - x[0]
d = np.max(np.abs(aux_d[1:]))
return np.divide(ln, np.add(ln, np.log10(np.divide(d, ll))))
@jit('UniTuple(float64, 2 )(float64[:], float64[:])', nopython=True)
 def _linear_regression(x, y):

 n_times = x.size
sx2 = 0
sx = 0
sy = 0
sxy = 0
for j in range(n_times):
sx2 += x[j] ** 2
sx += x[j]
sxy += x[j] * y[j]
sy += y[j]
den = n_times * sx2 -(sx ** 2)
 num = n_times * sxy - sx * sy
slope = num / den
intercept = np.mean(y)- slope * np.mean(x)
 return slope, intercept
@jit('i8[:](f8, f8, f8)', nopython=True)
 def _log_n(min_n, max_n, factor):

 max_i = int(floor(log(1.0 * max_n / min_n)/ log(factor)))
 ns = [min_n]
for i in range(max_i + 1):
n = int(floor(min_n *(factor ** i)))
 if n > ns[-1]:
ns.append(n)
 return np.array(ns, dtype=np.int64)
 @jit('float64(float64[:], int32)')
 def _higuchi_fd(x, kmax):

 n_times = x.size
lk = np.empty(kmax)
 x_reg = np.empty(kmax)
 y_reg = np.empty(kmax)
 for k in range(1, kmax + 1):
lm = np.empty(( k,))
for m in range(k):
ll = 0
n_max = floor(( n_times - m - 1)/ k)
 n_max = int(n_max)
 for j in range(1, n_max):
ll += abs(x[m + j * k] - x[m +(j - 1)* k])
 ll /= k
ll *=(n_times - 1)/(k * n_max)
 lm[m] = ll
m_lm = 0
for m in range(k):
m_lm += lm[m]
m_lm /= k
lk[k - 1] = m_lm
x_reg[k - 1] = log(1./ k)
 y_reg[k - 1] = log(m_lm)
 higuchi, _ = _linear_regression(x_reg, y_reg)
 return higuchi
def higuchi_fd(x, kmax=10):

 x = np.asarray(x, dtype=np.float64)
 kmax = int(kmax)
 return _higuchi_fd(x, kmax )
number_feature_cols = ['num_answer_user', 'num_question_user']
corr_df = calc_corr(train, target_cols, number_feature_cols )<import_modules>
def _embed(x, order=3, delay=1):

 N = len(x)
 if order * delay > N:
raise ValueError("Error: order * delay should be lower than x.size")
 if delay < 1:
raise ValueError("Delay has to be at least 1.")
 if order < 2:
raise ValueError("Order has to be at least 2.")
 Y = np.zeros(( order, N -(order - 1)* delay))
for i in range(order):
Y[i] = x[i * delay:i * delay + Y.shape[1]]
return Y.T
all = ['perm_entropy', 'spectral_entropy', 'svd_entropy', 'app_entropy',
'sample_entropy']
def perm_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 ran_order = range(order)
 hashmult = np.power(order, ran_order)
 sorted_idx = _embed(x, order=order, delay=delay ).argsort(kind='quicksort')
 hashval =(np.multiply(sorted_idx, hashmult)).sum(1)
 _, c = np.unique(hashval, return_counts=True)
 p = np.true_divide(c, c.sum())
 pe = -np.multiply(p, np.log2(p)).sum()
if normalize:
pe /= np.log2(factorial(order))
return pe
def spectral_entropy(x, sf, method='fft', nperseg=None, normalize=False):

 x = np.array(x)
 if method == 'fft':
_, psd = periodogram(x, sf)
 elif method == 'welch':
_, psd = welch(x, sf, nperseg=nperseg)
 psd_norm = np.divide(psd, psd.sum())
 se = -np.multiply(psd_norm, np.log2(psd_norm)).sum()
if normalize:
se /= np.log2(psd_norm.size)
 return se
def svd_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 mat = _embed(x, order=order, delay=delay)
 W = np.linalg.svd(mat, compute_uv=False)
 W /= sum(W)
 svd_e = -np.multiply(W, np.log2(W)).sum()
if normalize:
svd_e /= np.log2(order)
 return svd_e
def _app_samp_entropy(x, order, metric='chebyshev', approximate=True):

 _all_metrics = KDTree.valid_metrics
if metric not in _all_metrics:
raise ValueError('The given metric(%s)is not valid.The valid '
'metric names are: %s' %(metric, _all_metrics))
phi = np.zeros(2)
 r = 0.2 * np.std(x, axis=-1, ddof=1)
 _emb_data1 = _embed(x, order, 1)
 if approximate:
emb_data1 = _emb_data1
else:
emb_data1 = _emb_data1[:-1]
count1 = KDTree(emb_data1, metric=metric ).query_radius(emb_data1, r,
count_only=True
).astype(np.float64)
 emb_data2 = _embed(x, order + 1, 1)
 count2 = KDTree(emb_data2, metric=metric ).query_radius(emb_data2, r,
count_only=True
).astype(np.float64)
 if approximate:
phi[0] = np.mean(np.log(count1 / emb_data1.shape[0]))
phi[1] = np.mean(np.log(count2 / emb_data2.shape[0]))
else:
phi[0] = np.mean(( count1 - 1)/(emb_data1.shape[0] - 1))
phi[1] = np.mean(( count2 - 1)/(emb_data2.shape[0] - 1))
return phi
@jit('f8(f8[:], i4, f8)', nopython=True)
 def _numba_sampen(x, mm=2, r=0.2):

 n = x.size
n1 = n - 1
mm += 1
mm_dbld = 2 * mm
r *= x.std()
run = [0] * n
run1 = run[:]
r1 = [0] *(n * mm_dbld)
 a = [0] * mm
b = a[:]
p = a[:]
for i in range(n1):
nj = n1 - i
for jj in range(nj):
j = jj + i + 1
if abs(x[j] - x[i])< r:
run[jj] = run1[jj] + 1
m1 = mm if mm < run[jj] else run[jj]
for m in range(m1):
a[m] += 1
if j < n1:
b[m] += 1
else:
run[jj] = 0
for j in range(mm_dbld):
run1[j] = run[j]
r1[i + n * j] = run[j]
if nj > mm_dbld - 1:
for j in range(mm_dbld, nj):
run1[j] = run[j]
m = mm - 1
while m > 0:
b[m] = b[m - 1]
m -= 1
b[0] = n * n1 / 2
a = np.array([float(aa)for aa in a])
 b = np.array([float(bb)for bb in b])
 p = np.true_divide(a, b)
 return -log(p[-1])
 def app_entropy(x, order=2, metric='chebyshev'):

 phi = _app_samp_entropy(x, order=order, metric=metric, approximate=True)
 return np.subtract(phi[0], phi[1])
 def sample_entropy(x, order=2, metric='chebyshev'):

 x = np.asarray(x, dtype=np.float64)
 if metric == 'chebyshev' and x.size < 5000:
return _numba_sampen(x, mm=order, r=0.2)
 else:
phi = _app_samp_entropy(x, order=order, metric=metric,
approximate=False)
 return -np.log(np.divide(phi[1], phi[0]))
def validate_case_matches_checkpoint(do_lower_case, init_checkpoint):

 if not init_checkpoint:
return
m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint)
 if m is None:
return
model_name = m.group(1)
 lower_models = [
"uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12",
"multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12"
]
cased_models = [
"cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16",
"multi_cased_L-12_H-768_A-12"
]
is_bad_config = False
if model_name in lower_models and not do_lower_case:
is_bad_config = True
actual_flag = "False"
case_name = "lowercased"
opposite_flag = "True"
if model_name in cased_models and do_lower_case:
is_bad_config = True
actual_flag = "True"
case_name = "cased"
opposite_flag = "False"
if is_bad_config:
raise ValueError(
 "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`."
"However, `%s` seems to be a %s model, so you "
"should pass in `--do_lower_case=%s` so that the fine-tuning matches "
"how the model was pre-training.If this error is wrong, please "
"just comment out this check." %(actual_flag, init_checkpoint,
model_name, case_name, opposite_flag))
def convert_to_unicode(text):

 if six.PY3:
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 elif six.PY2:
if isinstance(text, str):
return text.decode("utf-8", "ignore")
 elif isinstance(text, unicode):
return text
else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 else:
raise ValueError("Not running on Python2 or Python 3?")
 def printable_text(text):

 if six.PY3:
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 elif six.PY2:
if isinstance(text, str):
return text
elif isinstance(text, unicode):
return text.encode("utf-8")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 else:
raise ValueError("Not running on Python2 or Python 3?")
 def load_vocab(vocab_file):

 vocab = collections.OrderedDict()
index = 0
with tf.io.gfile.GFile(vocab_file, "r")as reader:
while True:
token = convert_to_unicode(reader.readline())
 if not token:
break
token = token.strip()
vocab[token] = index
index += 1
return vocab
def convert_by_vocab(vocab, items):

 output = []
for item in items:
output.append(vocab[item])
 return output
def convert_tokens_to_ids(vocab, tokens):
return convert_by_vocab(vocab, tokens)
 def convert_ids_to_tokens(inv_vocab, ids):
return convert_by_vocab(inv_vocab, ids)
 def whitespace_tokenize(text):

 text = text.strip()
if not text:
return []
tokens = text.split()
return tokens
class FullTokenizer(object):

 def __init__(self, vocab_file, do_lower_case=True):
self.vocab = load_vocab(vocab_file)
 self.inv_vocab = {v: k for k, v in self.vocab.items() }
self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
 self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
 def tokenize(self, text):
split_tokens = []
for token in self.basic_tokenizer.tokenize(text):
for sub_token in self.wordpiece_tokenizer.tokenize(token):
split_tokens.append(sub_token)
 return split_tokens
def convert_tokens_to_ids(self, tokens):
return convert_by_vocab(self.vocab, tokens)
 def convert_ids_to_tokens(self, ids):
return convert_by_vocab(self.inv_vocab, ids)
 class BasicTokenizer(object):

 def __init__(self, do_lower_case=True):

 self.do_lower_case = do_lower_case
def tokenize(self, text):

 text = convert_to_unicode(text)
 text = self._clean_text(text)
 text = self._tokenize_chinese_chars(text)
 orig_tokens = whitespace_tokenize(text)
 split_tokens = []
for token in orig_tokens:
if self.do_lower_case:
token = token.lower()
token = self._run_strip_accents(token)
 split_tokens.extend(self._run_split_on_punc(token))
output_tokens = whitespace_tokenize(" ".join(split_tokens))
return output_tokens
def _run_strip_accents(self, text):

 text = unicodedata.normalize("NFD", text)
 output = []
for char in text:
cat = unicodedata.category(char)
 if cat == "Mn":
continue
output.append(char)
 return "".join(output)
 def _run_split_on_punc(self, text):

 chars = list(text)
 i = 0
start_new_word = True
output = []
while i < len(chars):
char = chars[i]
if _is_punctuation(char):
output.append([char])
 start_new_word = True
else:
if start_new_word:
output.append([])
 start_new_word = False
output[-1].append(char)
 i += 1
return ["".join(x)for x in output]
def _tokenize_chinese_chars(self, text):

 output = []
for char in text:
cp = ord(char)
 if self._is_chinese_char(cp):
output.append(" ")
 output.append(char)
 output.append(" ")
 else:
output.append(char)
 return "".join(output)
 def _is_chinese_char(self, cp):

 if(( cp >= 0x4E00 and cp <= 0x9FFF)or 
(cp >= 0x3400 and cp <= 0x4DBF)or 
(cp >= 0x20000 and cp <= 0x2A6DF)or 
(cp >= 0x2A700 and cp <= 0x2B73F)or 
(cp >= 0x2B740 and cp <= 0x2B81F)or 
(cp >= 0x2B820 and cp <= 0x2CEAF)or 
(cp >= 0xF900 and cp <= 0xFAFF)or 
(cp >= 0x2F800 and cp <= 0x2FA1F)) :
return True
return False
def _clean_text(self, text):

 output = []
for char in text:
cp = ord(char)
 if cp == 0 or cp == 0xfffd or _is_control(char):
continue
if _is_whitespace(char):
output.append(" ")
 else:
output.append(char)
 return "".join(output)
 class WordpieceTokenizer(object):

 def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200):
self.vocab = vocab
self.unk_token = unk_token
self.max_input_chars_per_word = max_input_chars_per_word
def tokenize(self, text):

 text = convert_to_unicode(text)
 output_tokens = []
for token in whitespace_tokenize(text):
chars = list(token)
 if len(chars)> self.max_input_chars_per_word:
output_tokens.append(self.unk_token)
 continue
is_bad = False
start = 0
sub_tokens = []
while start < len(chars):
end = len(chars)
 cur_substr = None
while start < end:
substr = "".join(chars[start:end])
 if start > 0:
substr = "
if substr in self.vocab:
cur_substr = substr
break
end -= 1
if cur_substr is None:
is_bad = True
break
sub_tokens.append(cur_substr)
 start = end
if is_bad:
output_tokens.append(self.unk_token)
 else:
output_tokens.extend(sub_tokens)
 return output_tokens
def _is_whitespace(char):

 , and \r are technically contorl characters but we treat them
if char == " " or char == "\t" or char == "
" or char == "\r":
return True
cat = unicodedata.category(char)
 if cat == "Zs":
return True
return False
def _is_control(char):

 if char == "\t" or char == "
" or char == "\r":
return False
cat = unicodedata.category(char)
 if cat in("Cc", "Cf"):
return True
return False
def _is_punctuation(char):

 cp = ord(char)
 if(( cp >= 33 and cp <= 47)or(cp >= 58 and cp <= 64)or 
(cp >= 91 and cp <= 96)or(cp >= 123 and cp <= 126)) :
return True
cat = unicodedata.category(char)
 if cat.startswith("P"):
return True
return False<set_options>
def entropy_and_fractal_dim(x):
return [perm_entropy(x), svd_entropy(x), app_entropy(x), sample_entropy(x), petrosian_fd(x), katz_fd(x), higuchi_fd(x)]
np.set_printoptions(suppress=True )<define_variables>
signals = X.reshape(( len(X), X.shape[1]*X.shape[2]))
features = []
for signal in signals:
features.append(entropy_and_fractal_dim(signal))
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True )<define_variables>
features = np.array(features ).reshape(( len(features), 7))
scaler = MinMaxScaler(feature_range=(0, 1))
scaler.fit(features)
 features = scaler.transform(features )
MAX_SEQUENCE_LENGTH = 512<string_transform>
def model_lstm(input_shape, feat_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
feat = Input(shape=(feat_shape[1],))
bi_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True), merge_mode='concat' )(inp)
 bi_gru = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat' )(bi_lstm)
 attention = Attention(input_shape[1] )(bi_gru)
 x = concatenate([attention, feat], axis=1)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp, feat], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=10 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_feat, train_y, val_X, val_feat, val_y = X[train_idx], features[train_idx], y[train_idx], X[val_idx], features[val_idx], y[val_idx]
model = model_lstm(train_X.shape, features.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit([train_X, train_feat], train_y, batch_size=128, epochs=50, validation_data=([val_X, val_feat], val_y), callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict([val_X, val_feat], batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 model.summary()
return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<define_variables>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(
 K.variable(y_true.astype("float64")) , K.variable(( y_proba > threshold ).astype("float64"))))
if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
output_categories = list(train.columns[11:41])
 input_categories = list(train.columns[[1,2,5]])
<split>
optimal_values = threshold_search(y_val, preds_val)
 best_threshold = optimal_values['threshold']
best_score = optimal_values['matthews_correlation']
gkf = GroupKFold(n_splits=10 ).split(X=train.question_body, groups=train.question_body)
 outputs = compute_output_arrays(train, target_cols)
 inputs = compute_input_arays(train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<load_pretrained>
print("Optimal Threshold : " + str(best_threshold))
print("Best Matthews Correlation : " + str(best_score))
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold <= 6:
K.clear_session()
model = bert_model()
print(f'/kaggle/input/bertbase/bert-base-{fold}-4.h5py')
 model.load_weights(f'/kaggle/input/bertbase/bert-base-{fold}-4.h5py')
 preds = model.predict(test_inputs,batch_size=8, verbose=1)
 histories.append(preds)
 if fold < 10 and fold >=7:
print(f'/kaggle/input/bertbaseextra/bert-base-{fold}-4.h5py')
 model.load_weights(f'/kaggle/input/bertbaseextra/bert-base-{fold}-4.h5py')
 preds = model.predict(test_inputs,batch_size=8, verbose=1)
 histories.append(preds )<define_variables>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
test_predictions_google_qa = histories<normalization>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape<save_to_csv>
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
sample_submission.iloc[:, 1:] = test_preds_google_qa
sample_submission.to_csv('submission.csv', index=False )<set_options>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape
np.set_printoptions(suppress=True)
 print(tf.__version__ )<install_modules>
signals = X_test_input.reshape(( len(X_test_input), X_test_input.shape[1]*X_test_input.shape[2]))
features_test = []
for signal in signals:
features_test.append(entropy_and_fractal_dim(signal))
!pip install.. /input/sacremoses > /dev/null
sys.path.insert(0, ".. /input/transformers/")
<load_pretrained>
features_test = np.array(features_test ).reshape(( len(features_test), 7))
features_test = scaler.transform(features_test)
 np.save("features_test.npy",features_test )
PATH = '.. /input/google-quest-challenge/'
core = 'bert'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
WEIGHTS_FILES_10 = sorted(glob('.. /input/bertbasetriplet5cv/*.h5'))
WEIGHTS_FILES_6_NOES = sorted(glob('.. /input/bert-base-5cv/*.h5'))
WEIGHTS_FILES_6 = sorted(glob('.. /input/bert-base-double-5cv-early-stop/*.h5'))
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories)
 print('
trained weights files(10 inputs):
\t', WEIGHTS_FILES_10)
 print('
trained weights files(6 inputs, without early stop):
\t', WEIGHTS_FILES_6_NOES)
 print('
trained weights files(6 inputs):
\t', WEIGHTS_FILES_6 )<categorify>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 input_ids_qa, input_masks_qa, input_segments_qa = return_id(
 title + ' ' + question, answer, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a,
input_ids_qa, input_masks_qa, input_segments_qa]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
input_ids_qa, input_masks_qa, input_segments_qa = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
inputs = _convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 ids_q, masks_q, segments_q = inputs[:3]
ids_a, masks_a, segments_a = inputs[3:6]
ids_qa, masks_qa, segments_qa = inputs[6:]
input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 input_ids_qa.append(ids_qa)
 input_masks_qa.append(masks_qa)
 input_segments_qa.append(segments_qa)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32),
np.asarray(input_ids_qa, dtype=np.int32),
np.asarray(input_masks_qa, dtype=np.int32),
np.asarray(input_segments_qa, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<feature_engineering>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict([X_test_input, features_test], batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )
def split_sentence(sentence):
return re.split('[\.\?]+\s*', sentence)
 def count_wps(sentence):
sentence_list = split_sentence(sentence)
 w_per_s = np.mean([len(sentence.split())for sentence in sentence_list
if len(sentence)> 0])
 return w_per_s
def build_count_features(df):
count_features = OrderedDict()
count_features['n_title_words'] = df.question_title.apply(lambda x: len(x.split()))
count_features['n_body_words'] = df.question_body.apply(lambda x: len(x.split()))
count_features['n_answer_words'] = df.answer.apply(lambda x: len(x.split()))
count_features['n_body_sentences'] = df.question_body.apply(lambda x: len(split_sentence(x)))
 count_features['n_answer_sentences'] = df.answer.apply(lambda x: len(split_sentence(x)))
 count_features['n_title_wps'] = df.question_title.apply(count_wps)
 count_features['n_body_wps'] = df.question_body.apply(count_wps)
 count_features['n_answer_wps'] = df.answer.apply(count_wps)
 df_count_features = pd.DataFrame(count_features)
 return df_count_features
def build_host_features(df, df_host_count, popular_host_thr):
def _convert(x):
if x not in df_host_count.index:
return 'other'
if df_host_count[x] > popular_host_thr:
return x
else:
return 'other'
return df.host.apply(_convert)
 def build_features(df, df_host_count, popular_host_thr):
return pd.concat(
 [build_count_features(df),
build_host_features(df, df_host_count, 100)],
axis=1 )<compute_test_metric>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<categorify><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()