Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<prepare_output>
exp_true = [9423, 9424, 9425,
9597, 9598, 9599,
10248, 10249, 10250,
11523, 11524, 11525,
12036, 12037, 12038,
12222, 12223, 12224,
13041, 13042, 13043,
14028, 14029, 14030,
14472, 14473, 14474,
15540, 15541, 15542,
17289, 17290, 17291,
17685, 17686, 17687,
19458, 19459, 19460,
22827, 22828, 22829,
22938, 22939, 22940,
23286, 23287, 23288,
24168, 24169, 24170,
25143, 25144, 25145,
26010, 26011, 26012]
exp_false = [8769, 8770, 8771,
9810, 9811, 9812,
11178, 11179, 11180,
12186, 12187, 12188,
13968, 13969, 13970,
14187, 14188, 14189,
15177, 15178, 15179,
16920, 16921, 16922,
18630, 18631, 18632,
18828, 18829, 18830,
19308, 19309, 19310,
20832, 20833, 20834,
26202, 26203, 26204]
final_predictions = np.mean(test_predictions, axis=0 )<load_pretrained>
import os
import numpy as np
import pandas as pd
import pyarrow.parquet as pq
from joblib import Parallel, delayed
from tqdm import tqdm
import tensorflow as tf
from tensorflow import set_random_seed
from keras.layers import *
from keras.models import Model
from keras import backend as K
from keras import optimizers
from keras.callbacks import *
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn.metrics import matthews_corrcoef
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.) 
<save_to_csv>
N_SPLITS = 5
sample_size = 800000
MAX_THREADS = 4
RANDOM_SEED = 2019
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )<import_modules>
np.random.seed(RANDOM_SEED)
 set_random_seed(RANDOM_SEED )
from sklearn.cross_validation import train_test_split
import lightgbm as lgb
import time<compute_train_metric>
class StatefullMCC(Layer):
def __init__(self, thresholds, **kwargs):
super(StatefullMCC, self ).__init__(**kwargs)
 self.thresholds = thresholds
self.stateful = True
self.name='matthews_correlation'
def reset_states(self):
K.get_session().run(tf.variables_initializer(self.local_variable))
def metric_variable(self, shape, dtype, validate_shape=True, name=None):
return tf.Variable(
 np.zeros(shape),
dtype=dtype,
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES],
validate_shape=validate_shape,
name=name, 
)
 def initialize_vars(self, y_true, y_pred):
self.tp = [None]*len(self.thresholds)
 self.tn = [None]*len(self.thresholds)
 self.fp = [None]*len(self.thresholds)
 self.fn = [None]*len(self.thresholds)
 for i,_ in enumerate(self.thresholds):
self.tp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tp%d'%i)
 self.tn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tn%d'%i)
 self.fp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fp%d'%i)
 self.fn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fn%d'%i)
 tp_op = tf.assign_add(self.tp[i], tf.count_nonzero(y_true * y_pred[i], axis=0))
self.add_update(tp_op)
 tn_op = tf.assign_add(self.tn[i], tf.count_nonzero(( 1-y_true)*(1-y_pred[i]), axis=0))
self.add_update(tn_op)
 fp_op = tf.assign_add(self.fp[i], tf.count_nonzero(( 1-y_true)*y_pred[i], axis=0))
self.add_update(fp_op)
 fn_op = tf.assign_add(self.fn[i], tf.count_nonzero(y_true*(1-y_pred[i]), axis=0))
self.add_update(fn_op)
 self.local_variable = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES)
 def __call__(self, y_true, y_pred):
rounded_preds = [K.cast(K.greater_equal(y_pred, t), 'float32')for t in self.thresholds]
self.initialize_vars(y_true, rounded_preds)
 mcc_vec = []
for i,_ in enumerate(self.thresholds):
num = tf.cast(( self.tp[i] * self.tn[i] - self.fp[i] * self.fn[i]), 'float32')
 den = K.sqrt(tf.cast(( self.tp[i] + self.fp[i])*(self.tp[i] + self.fn[i])*(self.tn[i] + self.fp[i])*(self.tn[i] + self.fn[i]), 'float32'))
mcc_vec.append(num/(den + tf.constant(K.epsilon())))
 return K.max(mcc_vec )
def lgb_modelfit_nocv(params,dtrain,dvalid,predictors,target='target',objective='binary',metrics='auc',feval=None,early_stopping_rounds=20,
num_boost_round=3000,verbose_eval=10,categorical_features=None):
lgb_params={
'boosting_type':'gbdt',
'objective':objective,
'metric':metrics,
'learning_rate':0.01,
'num_leaves':31,
'max_depth':-1,
'min_child_samples':20,
'max_bin':255,
'subsample':0.6,
'subsample_freq':0,
'colsample_bytree':0.3,
'min_child_weight':5,
'subsample_for_bin':200000,
'min_split_gain':0,
'reg_alpha':0,
'reg_lambda':0,
'nthread':8,
'verbose':0
}
lgb_params.update(params)
 print('preparing valildation datasets')
 xgtrain=lgb.Dataset(dtrain[predictors].values,label=dtrain[target].values,feature_name=predictors,
categorical_feature=categorical_features)
 xgvalid=lgb.Dataset(dvalid[predictors].values,label=dvalid[target].values,feature_name=predictors,
categorical_feature=categorical_features)
 evals_results={}
bst1=lgb.train(lgb_params,xgtrain,valid_sets=[xgtrain,xgvalid],valid_names=['trian','valid'],
evals_result=evals_results,num_boost_round=num_boost_round,early_stopping_rounds=early_stopping_rounds,
verbose_eval=10,feval=feval)
 n_estimators=bst1.best_iteration
print('Model Report')
 print('n_estimators:',n_estimators)
 print(metrics+':',evals_results['valid'][metrics][n_estimators-1])
 return bst1<load_from_csv>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.glorot_uniform(RANDOM_SEED)
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
path='.. /input/'
dtypes={
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('load train...')
 train_df = pd.read_csv(path+"train.csv",skiprows=range(1,149903891), nrows=40000000, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])
 print('load test...')
 test_df = pd.read_csv(path+"test.csv", dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'] )<concatenate>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
len_train = len(train_df)
 train_df=train_df.append(test_df )<data_type_conversions>
def get_features(dataset='train', split_parts=10):
if dataset == 'train':
cache_file = 'X.npy'
meta_file = '.. /input/metadata_train.csv'
elif dataset == 'test':
cache_file = 'X_test.npy'
meta_file = '.. /input/metadata_test.csv'
if os.path.isfile(cache_file):
X = np.load(cache_file)
 y = None
if dataset == 'train':
y = np.load('y.npy')
 else:
meta_df = pd.read_csv(meta_file)
 data_measurements = meta_df.pivot(index='id_measurement', columns='phase', values='signal_id')
 data_measurements = data_measurements.values
data_measurements = np.array_split(data_measurements, split_parts, axis=0)
 X = Parallel(n_jobs=min(split_parts, MAX_THREADS), verbose=1 )(delayed(prep_data )(p, dataset)for p in data_measurements)
 try:
y = meta_df.loc[meta_df['phase']==0, 'target'].values
except:
y = None
X = np.concatenate(X, axis=0)
 if dataset == 'train':
np.save("X.npy",X)
 np.save("y.npy",y)
 elif dataset == 'test':
np.save("X_test.npy",X)
 return X, y
del test_df
gc.collect()
print('data prep...')
 train_df['hour'] = pd.to_datetime(train_df.click_time ).dt.hour.astype('uint8')
 train_df['day'] = pd.to_datetime(train_df.click_time ).dt.day.astype('uint8')
 gc.collect()<groupby>
max_num = 127
min_num = -128
print('group by...')
 gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'} )<merge>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
ts_std =(ts - min_data)/(max_data - min_data)
 return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
print('merge...')
 train_df = train_df.merge(gp, on=['ip','day','hour'], how='left')
 print("vars and data type: ")
 train_df.info()<merge>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in tqdm(range(0, sample_size, bucket_size)) :
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
print('grouping by ip-app combination...')
 gp = train_df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
 train_df = train_df.merge(gp, on=['ip','app'], how='left')
 del gp
gc.collect()<merge>
def prep_data(signal_ids, dataset="train"):
signal_ids_all = np.concatenate(signal_ids)
 if dataset == "train":
praq_data = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
elif dataset == "test":
praq_data = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
else:
raise ValueError("Unknown dataset")
 X = []
for sids in tqdm(signal_ids):
data = praq_data[[str(s)for s in sids]].values.T
X_signal = [transform_ts(signal)for signal in data]
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 return X
print('grouping by ip-app-os combination...')
 gp = train_df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
 train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
 del gp
gc.collect()<merge>
%%time
X, y = get_features("train", split_parts=6 )
print('grouping by : ip_day_chl_var_hour')
 gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})
 train_df = train_df.merge(gp, on=['ip','day','channel'], how='left')
 del gp
gc.collect()<merge>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
print('grouping by : ip_app_os_var_hour')
 gp = train_df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})
 train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
 del gp
gc.collect()<merge>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
print('grouping by : ip_app_channel_var_day')
 gp = train_df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})
 train_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')
 del gp
gc.collect()<data_type_conversions>
tmp = meta_test.reset_index()
train_df['qty'] = train_df['qty'].astype('uint16')
 train_df['ip_app_count'] = train_df['ip_app_count'].astype('uint16')
 train_df['ip_app_os_count'] = train_df['ip_app_os_count'].astype('uint16' )<create_dataframe>
true_mask = tmp["signal_id"].isin(exp_true ).values[::3]
false_mask = tmp["signal_id"].isin(exp_false ).values[::3]
target = 'is_attributed'
predictors = ['app','device','os', 'channel', 'hour', 'qty',
'ip_tchan_count', 'ip_app_count',
'ip_app_os_count', 'ip_app_os_var',
'ip_app_channel_var_day']
categorical = ['app','device','os', 'channel', 'hour']
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id'].astype('int')
 gc.collect()<init_hyperparams>
%%time
MAX_THREADS = 4
X_test_input, _ = get_features("test" )
print("Training...")
 params = {
'learning_rate': 0.1,
'num_leaves': 1400,
'max_depth': 3,
'min_child_samples': 200,
'max_bin': 100,
'subsample':.7,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'min_child_weight': 0,
'scale_pos_weight':99
}<normalization>
true_X_test = X_test_input[true_mask]
false_X_test = X_test_input[false_mask]
bst = lgb_modelfit_nocv(params,
train_df,
val_df,
predictors,
target,
objective='binary',
metrics='auc',
early_stopping_rounds=50,
verbose_eval=True,
num_boost_round=400,
categorical_features=categorical )<save_to_csv>
XX = np.append(X, true_X_test, axis=0)
 X = np.append(XX, false_X_test, axis=0 )
del train_df
del val_df
gc.collect()
print("Predicting...")
 sub['is_attributed'] = bst.predict(test_df[predictors])
 print("writing...")
 sub.to_csv('sub_lgb_balanced99.csv',index=False)
 print("done...")
 print(sub.info() )<save_to_csv>
y = np.append(y, [1] *(len(exp_true)// 3)+ [0]*(len(exp_false)// 3), axis=0 )
<import_modules>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
init_glorot_uniform = initializers.glorot_uniform(seed=RANDOM_SEED)
 init_orthogonal = initializers.orthogonal(seed=RANDOM_SEED)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu", kernel_initializer=init_glorot_uniform )(x)
 x = Dense(1, activation="sigmoid", kernel_initializer=init_glorot_uniform )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[StatefullMCC(np.linspace(0.45,0.55,11)) ])
 return model
import pandas as pd
import numpy as np<load_from_csv>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in tqdm(enumerate(splits)) :
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=100, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
print("Reading the data...
 ")
 df1 = pd.read_csv('.. /input/r-lgbm-single-model-40m-rows-lb-0-9736/lgb_Usrnewness.csv')
 df2 = pd.read_csv('.. /input/simple-linear-stacking-with-ranks-lb-0-9760/sub_stacked.csv')
 df3 = pd.read_csv('.. /input/talkingdata-wordbatch-fm-ftrl-lb-0-9752/wordbatch_fm_ftrl.csv' )<define_variables>
def threshold_search(y_true, y_proba):
thresholds = np.linspace(0.0,1.0,101)
 scores = [matthews_corrcoef(y_true,(y_proba > t ).astype(np.uint8)) for t in thresholds]
best_idx = np.argmax(scores)
 return thresholds[best_idx], scores[best_idx]
models = { 'df1' : {
'name':'r-lgbm',
'score':97.36,
'df':df1 },
'df2' : {
'name':'linstack',
'score':97.60,
'df':df2 },
'df3' : {
'name':'wordbatch',
'score':97.52,
'df':df3 }
}<find_best_params>
best_threshold, best_score = threshold_search(y_val, preds_val)
 print(best_threshold, best_score )
isa_lg = 0
isa_hm = 0
print("Blending...
 ")
 for df in models.keys() :
isa_lg += np.log(models[df]['df'].is_attributed)
 isa_hm += 1/(models[df]['df'].is_attributed)
 isa_lg = np.exp(isa_lg/3)
 isa_hm = 1/isa_hm
print("Isa log
")
 print(isa_lg[:5])
 print()
print("Isa harmo
")
 print(isa_hm[:5] )<prepare_output>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
sub_log = pd.DataFrame()
sub_log['click_id'] = df1['click_id']
sub_log['is_attributed'] = isa_lg
sub_log.head()<prepare_output>
preds_test = []
for i in tqdm(range(N_SPLITS)) :
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
sub_hm = pd.DataFrame()
sub_hm['click_id'] = df1['click_id']
sub_hm['is_attributed'] = isa_hm
sub_hm.head()<save_to_csv>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<import_modules><EOS> 
submission['target'] = preds_test
submission.loc[submission["signal_id"].isin(exp_true), "target"] = 1
submission.loc[submission["signal_id"].isin(exp_false), "target"] = 0
submission.to_csv('submission.csv', index=False)
 submission.head()