Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<define_variables>
gc.enable()
cwd = os.getcwd()
train_path = '.. /input/'+'train.csv'
test_path = '.. /input/'+'test.csv'<define_variables>
warnings.simplefilter(action='ignore', category=FutureWarning )
train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time']<load_from_csv>
N_SPLITS = 5
sample_size = 800000
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('Loading the training data...')
 train = pd.read_csv(train_path, skiprows=range(1,84903891), dtype=dtypes, nrows=11000000,usecols=train_cols )<train_model>
def matthews_correlation(y_true, y_pred):

 y_true = K.cast(y_true, float)
 y_pred = K.cast(y_pred, float)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
len_train = len(train)
 print('The initial size of the train set is', len_train )<merge>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def df_featured(df):
print("Creating new time features: 'hour' and 'day'...")
 df['hour'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 df['day'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 gc.collect()
print("Feature Engineering
")
 print('1.Computing the number of clicks associated with a given IP address within each hour...')
 n_channel = df[['ip','day','hour','channel']].groupby(by=['ip','day',
'hour'])[['channel']].count().reset_index().rename(columns={'channel': 'n_channels'})
 print('Merging the channels data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','day','hour'], how='left')
 del n_channel
gc.collect()
print('2.Computing the number of clicks associated with a given IP address and app...')
 n_channel = df[['ip','app', 'channel']].groupby(by=['ip',
'app'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_count'})
 print('Merging the channels data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app'], how='left')
 del n_channel
gc.collect()
print('3.Computing the number of clicks associated with a given IP address, app, and os...')
 n_channel = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app',
'os'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_os_count'})
 print('Merging the channels data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'os'], how='left')
 del n_channel
gc.collect()
print('4.grouping by : ip_day_chl_var_hour.. ')
 n_channel = df[['ip','day','hour','channel']].groupby(by=['ip','day',
'channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})
 print('Merging the hour data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','day','channel'], how='left')
 del n_channel
gc.collect()
print('5.grouping by : ip_app_os_var_hour.. ')
 n_channel = df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app',
'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})
 print('Merging the hour data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'os'], how='left')
 del n_channel
gc.collect()
print('6.grouping by : ip_app_channel_var_day...')
 n_channel = df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app',
'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})
 print('Merging the day data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'channel'], how='left')
 del n_channel
gc.collect()
print('7.grouping by : ip_app_chl_mean_hour.. ')
 n_channel = df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app',
'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})
 print('Merging the meanhour data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'channel'], how='left')
 del n_channel
gc.collect()
print('8.group by : ip_hh_dev')
 n_channel = df[['ip', 'device', 'hour', 'day', 'channel']].groupby(by=['ip', 'device', 'day',
'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_dev'})
 print('Merging the channel data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','device','day','hour'], how='left')
 del n_channel
gc.collect()
df['n_channels'] = df['n_channels'].astype('uint16')
 df['ip_app_count'] = df['ip_app_count'].astype('uint16')
 df['ip_app_os_count'] = df['ip_app_os_count'].astype('uint16')
 df['ip_tchan_count'] = df['ip_tchan_count'].astype('float32')
 df['ip_app_os_var'] = df['ip_app_os_var'].astype('float32')
 df['ip_app_channel_var_day'] = df['ip_app_channel_var_day'].astype('float32')
 df['ip_app_channel_mean_hour'] = df['ip_app_channel_mean_hour'].astype('float32')
 df['nip_hh_dev'] = df['nip_hh_dev'].astype('uint16')
 df.drop(['ip','day'], axis=1, inplace=True)
 gc.collect()
return df<init_hyperparams>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
metrics = 'auc'
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric':metrics,
'learning_rate': 0.05,
'num_leaves': 31,
'max_depth': -1,
'min_child_samples': 20,
'max_bin': 255,
'subsample': 0.7,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'subsample_for_bin': 200000,
'min_child_weight': 5,
'min_split_gain': 0,
'nthread': 8,
'verbose': 0,
'scale_pos_weight':99.7,
'metric':metrics
}<data_type_conversions>
max_num = 127
min_num = -128
target = 'is_attributed'
train[target] = train[target].astype('uint8')
 predictors = ['app','device', 'os', 'channel', 'hour', 'n_channels', 'ip_app_count', 'ip_app_os_count',
'ip_tchan_count','ip_app_os_var','ip_app_channel_var_day','ip_app_channel_mean_hour','nip_hh_dev']
categorical = [ 'app', 'device', 'os', 'channel']
gc.collect()<train_model>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
VALIDATE = False
MAX_ROUNDS = 1000
EARLY_STOP = 50
OPT_ROUNDS = 1000
FULL_OUTFILE = 'sub_lgbm1.csv'
VALID_OUTFILE = 'sub_lgbm_v.csv'
if VALIDATE:
train, val_df = train_test_split(train, train_size=.95, shuffle=False)
 print("
Train Information ", train.info())
 print("
Val Information ", val_df.info())
 print("train size: ", len(train))
print("valid size: ", len(val_df))
gc.collect()
print("Training...
 ")
 num_boost_round=MAX_ROUNDS
early_stopping_rounds=EARLY_STOP
xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del train
gc.collect()
xgvalid = lgb.Dataset(val_df[predictors].values, label=val_df[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del val_df
gc.collect()
evals_results = {}
bst = lgb.train(lgb_params,
xgtrain,
valid_sets= [xgvalid],
valid_names=['valid'],
evals_result=evals_results,
num_boost_round=num_boost_round,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10,
feval=None)
 n_estimators = bst.best_iteration
print("
Model Report")
 print("n_estimators : ", n_estimators)
 print(metrics+":", evals_results['valid'][metrics][n_estimators-1])
 outfile = VALID_OUTFILE
del xgvalid
else:
print("train size: ", len(train))
gc.collect()
print("Training...")
 num_boost_round=OPT_ROUNDS
xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del train
gc.collect()
print("Dataset preparing done")
 bst = lgb.train(lgb_params,
xgtrain,
num_boost_round=num_boost_round,
verbose_eval=10,
feval=None)
 print("Traing done")
 outfile = FULL_OUTFILE
del xgtrain
gc.collect()
print('load test...')
 test_cols = ['ip','app','device','os', 'channel', 'click_time', 'click_id']
test_df = pd.read_csv(test_path, dtype=dtypes, usecols=test_cols)
 test_df = df_featured(test_df)
 gc.collect()
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id']
print("Predicting...")
 sub['is_attributed'] = bst.predict(test_df[predictors])
 print("writing...")
 sub.to_csv(outfile, index=False, float_format='%.9f')
 print("done...")
 print(sub.info() )<set_options>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
path = '.. /input/'
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('loading')
 train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_columns = ['click_id', 'ip', 'app', 'device', 'os', 'channel', 'click_time']
train_df = pd.read_csv(path+"train.csv", dtype=dtypes, skiprows = range(1, 131886954), usecols=train_columns)
 test_df = pd.read_csv(path+"test.csv", dtype=dtypes, usecols=test_columns)
 print('finished loading' )<data_type_conversions>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X_aug, y_aug = augment(X_temp,y_temp,t=2)
 X.append(X_aug)
 y.append(y_aug)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
def prep_data(d):
print('hour, day, wday.... ')
 d['hour'] = pd.to_datetime(d.click_time ).dt.hour.astype('uint8')
 d['day'] = pd.to_datetime(d.click_time ).dt.day.astype('uint8')
 d['wday'] = pd.to_datetime(d.click_time ).dt.dayofweek.astype('uint8')
 print('grouping by ip-day-hour combination.... ')
 gp = d[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})
 d = d.merge(gp, on=['ip', 'day', 'hour'], how='left')
 del gp; gc.collect()
print('group by ip-app combination.... ')
 gp = d[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
 d = d.merge(gp, on=['ip', 'app'], how='left')
 del gp; gc.collect()
print('group by ip-app-os combination.... ')
 gp = d[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
 d = d.merge(gp, on=['ip', 'app', 'os'], how='left')
 del gp; gc.collect()
print("vars and d type.... ")
 d['qty'] = d['qty'].astype('uint16')
 d['ip_app_count'] = d['ip_app_count'].astype('uint16')
 d['ip_app_os_count'] = d['ip_app_os_count'].astype('uint16')
 print("label encoding.... ")
 d[['app', 'device', 'os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)
 print('dropping')
 d.drop(['click_time', 'ip'], 1, inplace=True)
 return d<train_model>
def petrosian_fd(x):

 n = len(x)
 diff = np.ediff1d(x)
 N_delta =(diff[1:-1] * diff[0:-2] < 0 ).sum()
return np.log10(n)/(np.log10(n)+ np.log10(n /(n + 0.4 * N_delta)))
 def katz_fd(x):

 x = np.array(x)
 dists = np.abs(np.ediff1d(x))
ll = dists.sum()
ln = np.log10(np.divide(ll, dists.mean()))
aux_d = x - x[0]
d = np.max(np.abs(aux_d[1:]))
return np.divide(ln, np.add(ln, np.log10(np.divide(d, ll))))
@jit('UniTuple(float64, 2 )(float64[:], float64[:])', nopython=True)
 def _linear_regression(x, y):

 n_times = x.size
sx2 = 0
sx = 0
sy = 0
sxy = 0
for j in range(n_times):
sx2 += x[j] ** 2
sx += x[j]
sxy += x[j] * y[j]
sy += y[j]
den = n_times * sx2 -(sx ** 2)
 num = n_times * sxy - sx * sy
slope = num / den
intercept = np.mean(y)- slope * np.mean(x)
 return slope, intercept
@jit('i8[:](f8, f8, f8)', nopython=True)
 def _log_n(min_n, max_n, factor):

 max_i = int(floor(log(1.0 * max_n / min_n)/ log(factor)))
 ns = [min_n]
for i in range(max_i + 1):
n = int(floor(min_n *(factor ** i)))
 if n > ns[-1]:
ns.append(n)
 return np.array(ns, dtype=np.int64)
 @jit('float64(float64[:], int32)')
 def _higuchi_fd(x, kmax):

 n_times = x.size
lk = np.empty(kmax)
 x_reg = np.empty(kmax)
 y_reg = np.empty(kmax)
 for k in range(1, kmax + 1):
lm = np.empty(( k,))
for m in range(k):
ll = 0
n_max = floor(( n_times - m - 1)/ k)
 n_max = int(n_max)
 for j in range(1, n_max):
ll += abs(x[m + j * k] - x[m +(j - 1)* k])
 ll /= k
ll *=(n_times - 1)/(k * n_max)
 lm[m] = ll
m_lm = 0
for m in range(k):
m_lm += lm[m]
m_lm /= k
lk[k - 1] = m_lm
x_reg[k - 1] = log(1./ k)
 y_reg[k - 1] = log(m_lm)
 higuchi, _ = _linear_regression(x_reg, y_reg)
 return higuchi
def higuchi_fd(x, kmax=10):

 x = np.asarray(x, dtype=np.float64)
 kmax = int(kmax)
 return _higuchi_fd(x, kmax )
train_df = prep_data(train_df)
 test_df = prep_data(test_df)
 print("finished" )<split>
def _embed(x, order=3, delay=1):

 N = len(x)
 if order * delay > N:
raise ValueError("Error: order * delay should be lower than x.size")
 if delay < 1:
raise ValueError("Delay has to be at least 1.")
 if order < 2:
raise ValueError("Order has to be at least 2.")
 Y = np.zeros(( order, N -(order - 1)* delay))
for i in range(order):
Y[i] = x[i * delay:i * delay + Y.shape[1]]
return Y.T
all = ['perm_entropy', 'spectral_entropy', 'svd_entropy', 'app_entropy',
'sample_entropy']
def perm_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 ran_order = range(order)
 hashmult = np.power(order, ran_order)
 sorted_idx = _embed(x, order=order, delay=delay ).argsort(kind='quicksort')
 hashval =(np.multiply(sorted_idx, hashmult)).sum(1)
 _, c = np.unique(hashval, return_counts=True)
 p = np.true_divide(c, c.sum())
 pe = -np.multiply(p, np.log2(p)).sum()
if normalize:
pe /= np.log2(factorial(order))
return pe
def spectral_entropy(x, sf, method='fft', nperseg=None, normalize=False):

 x = np.array(x)
 if method == 'fft':
_, psd = periodogram(x, sf)
 elif method == 'welch':
_, psd = welch(x, sf, nperseg=nperseg)
 psd_norm = np.divide(psd, psd.sum())
 se = -np.multiply(psd_norm, np.log2(psd_norm)).sum()
if normalize:
se /= np.log2(psd_norm.size)
 return se
def svd_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 mat = _embed(x, order=order, delay=delay)
 W = np.linalg.svd(mat, compute_uv=False)
 W /= sum(W)
 svd_e = -np.multiply(W, np.log2(W)).sum()
if normalize:
svd_e /= np.log2(order)
 return svd_e
def _app_samp_entropy(x, order, metric='chebyshev', approximate=True):

 _all_metrics = KDTree.valid_metrics
if metric not in _all_metrics:
raise ValueError('The given metric(%s)is not valid.The valid '
'metric names are: %s' %(metric, _all_metrics))
phi = np.zeros(2)
 r = 0.2 * np.std(x, axis=-1, ddof=1)
 _emb_data1 = _embed(x, order, 1)
 if approximate:
emb_data1 = _emb_data1
else:
emb_data1 = _emb_data1[:-1]
count1 = KDTree(emb_data1, metric=metric ).query_radius(emb_data1, r,
count_only=True
).astype(np.float64)
 emb_data2 = _embed(x, order + 1, 1)
 count2 = KDTree(emb_data2, metric=metric ).query_radius(emb_data2, r,
count_only=True
).astype(np.float64)
 if approximate:
phi[0] = np.mean(np.log(count1 / emb_data1.shape[0]))
phi[1] = np.mean(np.log(count2 / emb_data2.shape[0]))
else:
phi[0] = np.mean(( count1 - 1)/(emb_data1.shape[0] - 1))
phi[1] = np.mean(( count2 - 1)/(emb_data2.shape[0] - 1))
return phi
@jit('f8(f8[:], i4, f8)', nopython=True)
 def _numba_sampen(x, mm=2, r=0.2):

 n = x.size
n1 = n - 1
mm += 1
mm_dbld = 2 * mm
r *= x.std()
run = [0] * n
run1 = run[:]
r1 = [0] *(n * mm_dbld)
 a = [0] * mm
b = a[:]
p = a[:]
for i in range(n1):
nj = n1 - i
for jj in range(nj):
j = jj + i + 1
if abs(x[j] - x[i])< r:
run[jj] = run1[jj] + 1
m1 = mm if mm < run[jj] else run[jj]
for m in range(m1):
a[m] += 1
if j < n1:
b[m] += 1
else:
run[jj] = 0
for j in range(mm_dbld):
run1[j] = run[j]
r1[i + n * j] = run[j]
if nj > mm_dbld - 1:
for j in range(mm_dbld, nj):
run1[j] = run[j]
m = mm - 1
while m > 0:
b[m] = b[m - 1]
m -= 1
b[0] = n * n1 / 2
a = np.array([float(aa)for aa in a])
 b = np.array([float(bb)for bb in b])
 p = np.true_divide(a, b)
 return -log(p[-1])
 def app_entropy(x, order=2, metric='chebyshev'):

 phi = _app_samp_entropy(x, order=order, metric=metric, approximate=True)
 return np.subtract(phi[0], phi[1])
 def sample_entropy(x, order=2, metric='chebyshev'):

 x = np.asarray(x, dtype=np.float64)
 if metric == 'chebyshev' and x.size < 5000:
return _numba_sampen(x, mm=order, r=0.2)
 else:
phi = _app_samp_entropy(x, order=order, metric=metric,
approximate=False)
 return -np.log(np.divide(phi[1], phi[0]))
RANDOM_SEED = 1
random.seed(RANDOM_SEED)
 np.random.seed(RANDOM_SEED)
 X_train, X_test = train_test_split(train_df, test_size=0.1, random_state=RANDOM_SEED)
 y_train = X_train['is_attributed']
X_train = X_train.drop(['is_attributed'], axis=1)
 y_test = X_test['is_attributed']
X_test = X_test.drop(['is_attributed'], axis=1 )<import_modules>
def entropy_and_fractal_dim(x):
return [perm_entropy(x), svd_entropy(x), app_entropy(x), sample_entropy(x), petrosian_fd(x), katz_fd(x), higuchi_fd(x),scipy.stats.kurtosis(x),denoised_std(x),signal_entropy(x),detail_coeffs_entropy(x, wavelet='db1')]+list(peaks(x))
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate
from keras.layers import BatchNormalization, SpatialDropout1D, Conv1D
from keras.callbacks import Callback
from keras.models import Model
from keras.optimizers import Adam<prepare_x_and_y>
signals = X.reshape(( len(X), X.shape[1]*X.shape[2]))
features = []
for signal in signals:
features.append(entropy_and_fractal_dim(signal))
def get_keras_data(dataset):
X = {
'app': np.array(dataset.app),
'ch': np.array(dataset.channel),
'dev': np.array(dataset.device),
'os': np.array(dataset.os),
'h': np.array(dataset.hour),
'd': np.array(dataset.day),
'wd': np.array(dataset.wday),
'qty': np.array(dataset.qty),
'c1': np.array(dataset.ip_app_count),
'c2': np.array(dataset.ip_app_os_count)
 }
return X
train = get_keras_data(X_train)
 test = get_keras_data(X_test )<define_variables>
features = np.array(features ).reshape(( len(features), 18))
scaler = MinMaxScaler(feature_range=(0, 1))
scaler.fit(features)
 features = scaler.transform(features )
max_app = np.max([train_df['app'].max() , test_df['app'].max() ])+1
max_ch = np.max([train_df['channel'].max() , test_df['channel'].max() ])+1
max_dev = np.max([train_df['device'].max() , test_df['device'].max() ])+1
max_os = np.max([train_df['os'].max() , test_df['os'].max() ])+1
max_h = np.max([train_df['hour'].max() , test_df['hour'].max() ])+1
max_d = np.max([train_df['day'].max() , test_df['day'].max() ])+1
max_wd = np.max([train_df['wday'].max() , test_df['wday'].max() ])+1
max_qty = np.max([train_df['qty'].max() , test_df['qty'].max() ])+1
max_c1 = np.max([train_df['ip_app_count'].max() , test_df['ip_app_count'].max() ])+1
max_c2 = np.max([train_df['ip_app_os_count'].max() , test_df['ip_app_os_count'].max() ])+1<choose_model_class>
def model_lstm(input_shape, feat_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
feat = Input(shape=(feat_shape[1],))
bi_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True), merge_mode='concat' )(inp)
 x = Dropout(0.23 )(bi_lstm)
 bi_gru = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat' )(x)
 attention = Attention(input_shape[1] )(bi_gru)
 x = concatenate([attention, feat], axis=1)
 x = Dropout(0.2 )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp, feat], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
emb_n = 50
in_app = Input(shape=[1], name = 'app')
 emb_app = Embedding(max_app, emb_n )(in_app)
 in_ch = Input(shape=[1], name = 'ch')
 emb_ch = Embedding(max_ch, emb_n )(in_ch)
 in_dev = Input(shape=[1], name = 'dev')
 emb_dev = Embedding(max_dev, emb_n )(in_dev)
 in_os = Input(shape=[1], name = 'os')
 emb_os = Embedding(max_os, emb_n )(in_os)
 in_h = Input(shape=[1], name = 'h')
 emb_h = Embedding(max_h, emb_n )(in_h)
 in_d = Input(shape=[1], name = 'd')
 emb_d = Embedding(max_d, emb_n )(in_d)
 in_wd = Input(shape=[1], name = 'wd')
 emb_wd = Embedding(max_wd, emb_n )(in_wd)
 in_qty = Input(shape=[1], name = 'qty')
 emb_qty = Embedding(max_qty, emb_n )(in_qty)
 in_c1 = Input(shape=[1], name = 'c1')
 emb_c1 = Embedding(max_c1, emb_n )(in_c1)
 in_c2 = Input(shape=[1], name = 'c2')
 emb_c2 = Embedding(max_c2, emb_n )(in_c2)
 fe = concatenate([(emb_app),(emb_ch),(emb_dev),(emb_os),(emb_h), 
(emb_d),(emb_wd),(emb_qty),(emb_c1),(emb_c2)] )<choose_model_class>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_feat, train_y, val_X, val_feat, val_y = X[train_idx], features[train_idx], y[train_idx], X[val_idx], features[val_idx], y[val_idx]
model = model_lstm(train_X.shape, features.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit([train_X, train_feat], train_y, batch_size=128, epochs=70, validation_data=([val_X, val_feat], val_y), callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict([val_X, val_feat], batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
s_dout = SpatialDropout1D(0.25 )(fe)
 fl1 = Flatten()(s_dout)
 conv1 = Conv1D(70, kernel_size=2, strides=1, padding='same' )(s_dout)
 fl2 = Flatten()(conv1)
 conv2 = Conv1D(70, kernel_size=4, strides=1, padding='same' )(s_dout)
 fl3 = Flatten()(conv2)
 concat = concatenate([(fl1),(fl2),(fl3)])
 x = Dropout(0.25 )(Dense(500, activation='relu' )(concat))
x = Dropout(0.35 )(Dense(750, activation='relu' )(x))
x = Dropout(0.45 )(Dense(1000, activation='relu' )(x))
outp = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[in_app,in_ch,in_dev,in_os,in_h,in_d,in_wd,in_qty,in_c1,in_c2], outputs=outp )<choose_model_class>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
batch_size = 50000
epochs = 1
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(train_df)/ batch_size)* epochs
lr_init, lr_fin = 0.0025, 0.0001
lr_decay = exp_decay(lr_init, lr_fin, steps)
 optimizer_adam = Adam(lr=0.0025, decay=lr_decay)
 model.compile(loss='binary_crossentropy', optimizer=optimizer_adam, metrics=['accuracy'] )<train_model>
optimal_values = threshold_search(y_val, preds_val)
 best_threshold = optimal_values['threshold']
best_score = optimal_values['matthews_correlation']
class_weight = {0:.01,1:.99}
model.fit(train, y_train, batch_size=batch_size, epochs=epochs, class_weight=class_weight, shuffle=True, validation_data=(test, y_test), verbose=1 )<predict_on_test>
print("Optimal Threshold : " + str(best_threshold))
print("Best Matthews Correlation : " + str(best_score))
predictions = model.predict(test, batch_size=batch_size, verbose=1 )<predict_on_test>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
ids = test_df['click_id']
test_df.drop('click_id', axis=1, inplace=True)
 t = get_keras_data(test_df)
 sub_preds = model.predict(t, batch_size=batch_size, verbose=1 )<save_to_csv>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
sub = pd.DataFrame()
sub['click_id'] = ids
sub['is_attributed'] = sub_preds
sub.to_csv('dl_sub.csv', index=False )<define_variables>
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[str(i)]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
VAL_FILE = '.. /input/training-and-validation-data-pickle/validation.pkl.gz'<define_variables>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape
almost_zero = 1e-10
almost_one = 1 - almost_zero<define_variables>
signals = X_test_input.reshape(( len(X_test_input), X_test_input.shape[1]*X_test_input.shape[2]))
features_test = []
for signal in signals:
features_test.append(entropy_and_fractal_dim(signal))
base_models = {
'lgb1 ': "Python LGBM based on Pranav Pandya's R version",
'wbftl': "anttip's Wordbatch FM-FTRL",
'nngpu': "Downampled Neural Network run on GPU"
}<define_variables>
features_test = np.array(features_test ).reshape(( len(features_test), 18))
features_test = scaler.transform(features_test)
 np.save("features_test.npy",features_test )
cvfiles = {
'lgb1 ': '.. /input/validate-pranav-lgb-model/pranav_lgb_val_nostop.csv',
'wbftl': '.. /input/validate-anttip-s-wordbatch-fm-ftrl-9711-version/wordbatch_fm_ftrl_val.csv',
'nngpu': '.. /input/gpu-validation/gpu_val1.csv'
}<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
subfiles = {
'lgb1 ': '.. /input/try-pranav-s-r-lgbm-in-python/sub_lgbm_r_to_python_nocv.csv',
'wbftl': '.. /input/anttip-s-wordbatch-fm-ftrl-9711-version/wordbatch_fm_ftrl.csv',
'nngpu': '.. /input/talkingdata-gpu-example-with-multiple-runs/gpu_test2.csv'
}<define_variables>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict([X_test_input, features_test], batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )
lbscores = {
'lgb1 ':.9694,
'wbftl':.9711,
'nngpu':.9678
}<prepare_x_and_y>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<train_model><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()