Kaggle competition: VSB Power Line Fault Detection
bert_base_cased_answer_models = create_bert_base_cased_answer_models()
bert_base_cased_answer_preds = predict(bert_base_cased_answer_models, test_loader )<set_options>
v_raw_train = pq.read_pandas('.. /input/vsb-power-line-fault-detection/train.parquet' ).to_pandas().values
meta_train = np.loadtxt('.. /input/vsb-power-line-fault-detection/metadata_train.csv', skiprows=1, delimiter=',')
 y_train = meta_train[:, 3].astype(bool)
 print(v_raw_train.shape )
del bert_base_cased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()<concatenate>
def compute_spectra(v_raw, *, m = 1000):

 percentile =(100, 99, 95, 0, 1, 5)
 n = v_raw.shape[1]
length = v_raw.shape[0] // m
n_spectra = len(percentile)
 mean_signal = np.zeros(( n, length), dtype='float32')
 percentile_spectra = np.zeros(( n, length, n_spectra), dtype='float32')
 print('computing spectra...', flush=True)
 for i in tqdm(range(n)) :
v = v_raw[:, i].astype('float32' ).reshape(-1, m)/ 128.0
mean = np.mean(v, axis=1)
 s = np.abs(np.percentile(v, percentile, axis=1)- mean)
 h = np.percentile(s, 5.0)
 s = np.maximum(0.0, s - h)
 mean_signal[i, :] = mean
percentile_spectra[i, :, :] = s.T
d = {}
d['mean'] = mean_signal
d['percentile'] = percentile_spectra
return d
spec_train = compute_spectra(v_raw_train)
 print('done.' )
bert_base_cased_question_answer_preds = np.concatenate([bert_base_cased_question_preds, bert_base_cased_answer_preds], axis=1 )<load_pretrained>
def max_windowed(spec, *, width=150, stride=10):

 n = spec.shape[0]
length = spec.shape[1]
nspec = spec.shape[2]
n_triplet = n // 3
max_spec3 = np.empty(( n_triplet, length, 3))
for i_triplet in range(n_triplet):
max_spec3[i_triplet, :, 0] = spec[3*i_triplet, :, 0]
max_spec3[i_triplet, :, 1] = spec[3*i_triplet + 1, :, 0]
max_spec3[i_triplet, :, 2] = spec[3*i_triplet + 2, :, 0]
x = tf.placeholder(tf.float32, [None, length, 3])
 K = np.ones(( width, 3, 1), dtype='float32')/ width
W_conv1 = tf.constant(K)
 h_conv1 = tf.nn.conv1d(x, W_conv1, stride=stride, padding='VALID')
 with tf.Session() as sess:
w = sess.run(h_conv1, feed_dict={x:max_spec3})
 imax = np.argmax(w[:, :, 0], axis=1)
 d = {}
d['w'] = w
d['ibegin'] = imax*stride
return d
peaks = max_windowed(spec_train['percentile'] )
test_loader, _ = get_test_loader(model_type="bert-base-uncased", content="Question", batch_size=32 )<predict_on_test>
def compute_features(v_raw, spec=None):

 if spec is None:
spec = compute_spectra(v_raw)
 v_spec = spec['percentile']
shape = v_spec.shape
n = shape[0]
length = shape[1]
nspec = shape[2]
n_triplet = n // 3
spec3 = np.empty(( n_triplet, length, nspec, 3))
for i_triplet in range(n_triplet):
spec3[i_triplet, :, :, 0] = v_spec[3*i_triplet, :, :]
spec3[i_triplet, :, :, 1] = v_spec[3*i_triplet + 1, :, :]
spec3[i_triplet, :, :, 2] = v_spec[3*i_triplet + 2, :, :]
width = 150
peaks = max_windowed(v_spec, width=width)
 n_feature4 = 3
X = np.empty(( n_triplet, n_feature4*nspec*3 + 3))
X4 = np.empty(( n_triplet, n_feature4, nspec, 3))
for i_triplet in range(n_triplet):
X4[i_triplet, 0, :, :] = np.max(spec3[i_triplet, :, :, :], axis=0)
 ibegin = peaks['ibegin'][i_triplet]
iend = ibegin + width
imid = ibegin + width // 2
X4[i_triplet, 1, :, :] = np.mean(spec3[i_triplet, ibegin:iend, :, :], axis=0)
 X4[i_triplet, 2, :, :] = np.max(spec3[i_triplet, ibegin:iend, :, :], axis=0)
 X[i_triplet, 0] = spec['mean'][3*i_triplet, imid]
X[i_triplet, 1] = spec['mean'][3*i_triplet + 1, imid]
X[i_triplet, 2] = spec['mean'][3*i_triplet + 2, imid]
shape = X4.shape
X[:, 3:] = X4.reshape(shape[0], shape[1]*shape[2]*shape[3])
 return X
X_all3 = compute_features(v_raw_train, spec_train)
 y_all3 = np.sum(y_train.reshape(-1, 3), axis=1)>= 2
print('Three phases are combined into one training data; the shapes are, therefore,')
 print(X_all3.shape, y_all3.shape )
bert_base_uncased_question_models = create_bert_base_uncased_question_models()
bert_base_uncased_question_preds = predict(bert_base_uncased_question_models, test_loader )<set_options>
knowledge_data = pd.read_csv('.. /input/vsb-knowledge-0744/sub_ens_v16.csv' )
del bert_base_uncased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()<load_pretrained>
if 'v_raw_train' in globals() :
del v_raw_train
id_test = np.loadtxt('.. /input/vsb-power-line-fault-detection/metadata_test.csv', skiprows=1, delimiter=',')[:, 0].astype(int)
 n_test = len(id_test)
 X_tests = []
n_subset = 4
nread = 0
for i_subset in range(n_subset):
ibegin = 8712 + 3*int(n_test // 3 *(i_subset/n_subset))
iend = 8712 + 3*int(n_test // 3 *(( i_subset + 1)/n_subset))
print('Loading %d/%d; signal_id %d - %d...' %(i_subset, n_subset, ibegin, iend))
v_raw_test = pq.read_pandas('.. /input/vsb-power-line-fault-detection/test.parquet',
columns=[str(i)for i in range(ibegin, iend)] ).to_pandas().values
nread += v_raw_test.shape[1]
X = compute_features(v_raw_test)
 X_tests.append(X)
 print('%d/%d test data processed.' %(nread, n_test))
del v_raw_test
X_test = np.concatenate(X_tests, axis=0)
 assert(X_test.shape[0] == id_test.shape[0] // 3)
 del X_tests
print('X_test computation done.shape', X_test.shape )
test_loader, _ = get_test_loader(model_type="bert-base-uncased", content="Answer", batch_size=32 )<predict_on_test>
len_train=len(X_all3)
 y_list=knowledge_data['target'].values
y_test=[]
for j in range(0,len(y_list),3):
y_test.append(y_list[j])
 y_test=np.asarray(y_test)
 del knowledge_data
print(X_all3.shape,y_all3.shape)
 print(X_test.shape,y_test.shape)
 X_all3=np.concatenate([X_all3,X_test])
 y_all3=np.concatenate([y_all3,y_test])
bert_base_uncased_answer_models = create_bert_base_uncased_answer_models()
bert_base_uncased_answer_preds = predict(bert_base_uncased_answer_models, test_loader )<set_options>
n_splits = 5
models = []
scores = np.zeros(n_splits)
 print('Training...')
 print('MCC training & cv')
 seeds=[0,42,1204,2019]
for seed in seeds:
splits = list(sklearn.model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed ).split(X_all3, y_all3))
for i,(idx_train, idx_cv)in enumerate(splits):
X_train = X_all3[idx_train, :]
y_train = y_all3[idx_train]
X_cv = X_all3[idx_cv, :]
y_cv = y_all3[idx_cv]
learning_rate = 0.006
model = CatBoostClassifier(learning_rate=learning_rate, od_type='IncToDec',
loss_function='Logloss', use_best_model=True, eval_metric='MCC')
 model.fit(X_train, y_train.astype(float), silent=True,
eval_set=(X_cv, y_cv.astype(float)))
 y_predict_train = model.predict(X_train)
 y_predict_cv = model.predict(X_cv)
 score_cv = sklearn.metrics.matthews_corrcoef(y_cv, y_predict_cv)
 models.append(model)
 scores[i] = score_cv
print('CV scores %.3f ± %.3f' %(np.mean(scores), np.std(scores)) )
del bert_base_uncased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()<concatenate>
bert_base_uncased_question_answer_preds = np.concatenate([bert_base_uncased_question_preds, bert_base_uncased_answer_preds], axis=1 )<load_pretrained>
y_test_probas = np.empty(( X_test.shape[0], n_splits*len(seeds)))
 for i, model in enumerate(models):
y_test_probas[:, i] = model.predict_proba(X_test)[:, 1]
y_test_proba = np.mean(y_test_probas, axis=1)
 y_submit = np.repeat(y_test_proba > 0.25, 3)
 print('Positive fraction %d/%d = %.3f' %(
 np.sum(y_submit), len(y_submit), np.sum(y_submit)/len(y_submit)) )
<predict_on_test><EOS> 
assert(len(id_test)== len(y_submit))
n = len(id_test)
 with open('submission.csv', 'w')as f:
f.write('signal_id,target
')
 for i in range(n):
f.write('%d,%d
' %(id_test[i], int(y_submit[i])))
 print('submission.csv written' )