Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<compute_test_metric>
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import *
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
from keras import activations
from keras import regularizers
from keras import initializers
from keras import constraints
from sklearn.preprocessing import MinMaxScaler
from numba import jit
from math import log, floor
from sklearn.neighbors import KDTree
from scipy.signal import periodogram, welch
from keras.engine import Layer
from keras.engine import InputSpec
from keras.objectives import categorical_crossentropy
from keras.objectives import sparse_categorical_crossentropy
def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean() )<load_from_csv>
N_SPLITS = 5
sample_size = 800000
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head(5 )<load_from_csv>
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 test.head(5 )<define_variables>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
train_null_columns_ovr50 = ['Alley','MiscFeature','Fence','PoolQC','FireplaceQu']
test_null_columns_ovr50 = ['MiscFeature','Fence','PoolQC','FireplaceQu','Alley']<define_variables>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
train_no_feature_columns = ['MasVnrType','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','GarageCond']
test_no_feature_columns = ['MasVnrType','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','GarageCond']<data_type_conversions>
max_num = 127
min_num = -128
for column in train_null_columns_ovr50:
train[column].fillna("NA",inplace=True)
 for column in test_null_columns_ovr50:
test[column].fillna("NA",inplace=True)
 for column in train_no_feature_columns:
train[column].fillna("NA",inplace=True)
 for column in test_no_feature_columns:
test[column].fillna("NA",inplace=True )<feature_engineering>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
train['GarageYrBlt'][train['GarageType']=='NA'] = 0
train['GarageYrBlt'][train['GarageYrBlt'].isnull() ] = train['YearBuilt'][train['GarageYrBlt'].isnull() ]
test['GarageYrBlt'][test['GarageType']=='NA'] = 0
test['GarageYrBlt'][test['GarageYrBlt'].isnull() ] = test['YearBuilt'][test['GarageYrBlt'].isnull() ]

 train['MasVnrArea'][train['MasVnrType']=='NA'] = 0
test['MasVnrArea'][test['MasVnrType']=='NA'] = 0

 test['TotalBsmtSF'][test['BsmtQual']=='NA'] = 0
test['GarageArea'][test['GarageArea'].isnull() ]= int(test.loc[test['GarageType']=='Detchd',['GarageArea']].mean())

test['GarageCars'][test['GarageQual']=='NA'] = 0

 test['SaleType'][test['SaleType'].isnull() ] = 'Oth'
test['Functional'][test['Functional'].isnull() ] = 'Typ'

 test['KitchenQual'][test['KitchenQual'].isnull() ] = 'TA'

 test['BsmtHalfBath'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtFullBath'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtUnfSF'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtFinSF2'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtFinSF1'][test['BsmtQual'] == 'NA'] = 0

 test['Exterior2nd'][test['Exterior2nd'].isnull() ] = 'Other'

 test['Exterior1st'][test['Exterior1st'].isnull() ] = 'Other'

 test['Utilities'][test['OverallQual']==6] = 'NoSewr'
test['Utilities'][test['OverallQual']==2] = 'ELO'

 train['Electrical'][train['Electrical'].isnull() ] = 'Mixed'<feature_engineering>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
def mean_of_neighborhood(df,neighborhood):
mean = df.loc[(df['Neighborhood']==neighborhood),'LotFrontage'].mean()
return mean
train_lot_frontage_mean = {neighborhood:mean_of_neighborhood(train,neighborhood)for neighborhood in list(train['Neighborhood'].unique())}
test_lot_frontage_mean = {neighborhood:mean_of_neighborhood(test,neighborhood)for neighborhood in list(test['Neighborhood'].unique())}
train['LotFrontage'] = train['LotFrontage'].fillna(train['Neighborhood'].map(train_lot_frontage_mean))
test['LotFrontage'] = test['LotFrontage'].fillna(test['Neighborhood'].map(test_lot_frontage_mean))<data_type_conversions>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
train['GarageYrBlt'] = train['GarageYrBlt'].astype(int)
 train['HouseAge'] = train.apply(lambda row: years_between(datetime.datetime(row['YearBuilt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearBuilt'] != 0 else 0,axis=1)
 train['RemodAddAge'] = train.apply(lambda row: years_between(datetime.datetime(row['YearRemodAdd'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearRemodAdd'] != 0 else 0,axis=1)
 train['GarageAge'] = train.apply(lambda row: years_between(datetime.datetime(row['GarageYrBlt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['GarageYrBlt'] != 0 else 0,axis=1)
 numerical_columns.extend(['HouseAge','RemodAddAge','GarageAge'])
 excluded_numerical_columns = ['YrSold','MoSold','GarageYrBlt','YearRemodAdd','YearBuilt']
numerical_columns = [x for x in numerical_columns if x not in excluded_numerical_columns]
train[['HouseAge','RemodAddAge','GarageAge']]<feature_engineering>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
test['GarageYrBlt'] = test['GarageYrBlt'].astype(int)
 test['HouseAge'] = test.apply(lambda row: years_between(datetime.datetime(row['YearBuilt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearBuilt'] != 0 else 0,axis=1)
 test['RemodAddAge'] = test.apply(lambda row: years_between(datetime.datetime(row['YearRemodAdd'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearRemodAdd'] != 0 else 0,axis=1)
 test['GarageAge'] = test.apply(lambda row: years_between(datetime.datetime(row['GarageYrBlt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['GarageYrBlt'] != 0 else 0,axis=1)
 test[['HouseAge','RemodAddAge','GarageAge']]<load_from_csv>
def petrosian_fd(x):

 n = len(x)
 diff = np.ediff1d(x)
 N_delta =(diff[1:-1] * diff[0:-2] < 0 ).sum()
return np.log10(n)/(np.log10(n)+ np.log10(n /(n + 0.4 * N_delta)))
 def katz_fd(x):

 x = np.array(x)
 dists = np.abs(np.ediff1d(x))
ll = dists.sum()
ln = np.log10(np.divide(ll, dists.mean()))
aux_d = x - x[0]
d = np.max(np.abs(aux_d[1:]))
return np.divide(ln, np.add(ln, np.log10(np.divide(d, ll))))
@jit('UniTuple(float64, 2 )(float64[:], float64[:])', nopython=True)
 def _linear_regression(x, y):

 n_times = x.size
sx2 = 0
sx = 0
sy = 0
sxy = 0
for j in range(n_times):
sx2 += x[j] ** 2
sx += x[j]
sxy += x[j] * y[j]
sy += y[j]
den = n_times * sx2 -(sx ** 2)
 num = n_times * sxy - sx * sy
slope = num / den
intercept = np.mean(y)- slope * np.mean(x)
 return slope, intercept
@jit('i8[:](f8, f8, f8)', nopython=True)
 def _log_n(min_n, max_n, factor):

 max_i = int(floor(log(1.0 * max_n / min_n)/ log(factor)))
 ns = [min_n]
for i in range(max_i + 1):
n = int(floor(min_n *(factor ** i)))
 if n > ns[-1]:
ns.append(n)
 return np.array(ns, dtype=np.int64)
 @jit('float64(float64[:], int32)')
 def _higuchi_fd(x, kmax):

 n_times = x.size
lk = np.empty(kmax)
 x_reg = np.empty(kmax)
 y_reg = np.empty(kmax)
 for k in range(1, kmax + 1):
lm = np.empty(( k,))
for m in range(k):
ll = 0
n_max = floor(( n_times - m - 1)/ k)
 n_max = int(n_max)
 for j in range(1, n_max):
ll += abs(x[m + j * k] - x[m +(j - 1)* k])
 ll /= k
ll *=(n_times - 1)/(k * n_max)
 lm[m] = ll
m_lm = 0
for m in range(k):
m_lm += lm[m]
m_lm /= k
lk[k - 1] = m_lm
x_reg[k - 1] = log(1./ k)
 y_reg[k - 1] = log(m_lm)
 higuchi, _ = _linear_regression(x_reg, y_reg)
 return higuchi
def higuchi_fd(x, kmax=10):

 x = np.asarray(x, dtype=np.float64)
 kmax = int(kmax)
 return _higuchi_fd(x, kmax )
us_inflation_rates = pd.read_csv(".. /input/us-inflation-rates/US_Inflation_Rates.csv")
 print("US Inflation rates head:")
 print(us_inflation_rates.head())
 print("US Inflation rates tail:")
 print(us_inflation_rates.tail() )<feature_engineering>
def _embed(x, order=3, delay=1):

 N = len(x)
 if order * delay > N:
raise ValueError("Error: order * delay should be lower than x.size")
 if delay < 1:
raise ValueError("Delay has to be at least 1.")
 if order < 2:
raise ValueError("Order has to be at least 2.")
 Y = np.zeros(( order, N -(order - 1)* delay))
for i in range(order):
Y[i] = x[i * delay:i * delay + Y.shape[1]]
return Y.T
all = ['perm_entropy', 'spectral_entropy', 'svd_entropy', 'app_entropy',
'sample_entropy']
def perm_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 ran_order = range(order)
 hashmult = np.power(order, ran_order)
 sorted_idx = _embed(x, order=order, delay=delay ).argsort(kind='quicksort')
 hashval =(np.multiply(sorted_idx, hashmult)).sum(1)
 _, c = np.unique(hashval, return_counts=True)
 p = np.true_divide(c, c.sum())
 pe = -np.multiply(p, np.log2(p)).sum()
if normalize:
pe /= np.log2(factorial(order))
return pe
def spectral_entropy(x, sf, method='fft', nperseg=None, normalize=False):

 x = np.array(x)
 if method == 'fft':
_, psd = periodogram(x, sf)
 elif method == 'welch':
_, psd = welch(x, sf, nperseg=nperseg)
 psd_norm = np.divide(psd, psd.sum())
 se = -np.multiply(psd_norm, np.log2(psd_norm)).sum()
if normalize:
se /= np.log2(psd_norm.size)
 return se
def svd_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 mat = _embed(x, order=order, delay=delay)
 W = np.linalg.svd(mat, compute_uv=False)
 W /= sum(W)
 svd_e = -np.multiply(W, np.log2(W)).sum()
if normalize:
svd_e /= np.log2(order)
 return svd_e
def _app_samp_entropy(x, order, metric='chebyshev', approximate=True):

 _all_metrics = KDTree.valid_metrics
if metric not in _all_metrics:
raise ValueError('The given metric(%s)is not valid.The valid '
'metric names are: %s' %(metric, _all_metrics))
phi = np.zeros(2)
 r = 0.2 * np.std(x, axis=-1, ddof=1)
 _emb_data1 = _embed(x, order, 1)
 if approximate:
emb_data1 = _emb_data1
else:
emb_data1 = _emb_data1[:-1]
count1 = KDTree(emb_data1, metric=metric ).query_radius(emb_data1, r,
count_only=True
).astype(np.float64)
 emb_data2 = _embed(x, order + 1, 1)
 count2 = KDTree(emb_data2, metric=metric ).query_radius(emb_data2, r,
count_only=True
).astype(np.float64)
 if approximate:
phi[0] = np.mean(np.log(count1 / emb_data1.shape[0]))
phi[1] = np.mean(np.log(count2 / emb_data2.shape[0]))
else:
phi[0] = np.mean(( count1 - 1)/(emb_data1.shape[0] - 1))
phi[1] = np.mean(( count2 - 1)/(emb_data2.shape[0] - 1))
return phi
@jit('f8(f8[:], i4, f8)', nopython=True)
 def _numba_sampen(x, mm=2, r=0.2):

 n = x.size
n1 = n - 1
mm += 1
mm_dbld = 2 * mm
r *= x.std()
run = [0] * n
run1 = run[:]
r1 = [0] *(n * mm_dbld)
 a = [0] * mm
b = a[:]
p = a[:]
for i in range(n1):
nj = n1 - i
for jj in range(nj):
j = jj + i + 1
if abs(x[j] - x[i])< r:
run[jj] = run1[jj] + 1
m1 = mm if mm < run[jj] else run[jj]
for m in range(m1):
a[m] += 1
if j < n1:
b[m] += 1
else:
run[jj] = 0
for j in range(mm_dbld):
run1[j] = run[j]
r1[i + n * j] = run[j]
if nj > mm_dbld - 1:
for j in range(mm_dbld, nj):
run1[j] = run[j]
m = mm - 1
while m > 0:
b[m] = b[m - 1]
m -= 1
b[0] = n * n1 / 2
a = np.array([float(aa)for aa in a])
 b = np.array([float(bb)for bb in b])
 p = np.true_divide(a, b)
 return -log(p[-1])
 def app_entropy(x, order=2, metric='chebyshev'):

 phi = _app_samp_entropy(x, order=order, metric=metric, approximate=True)
 return np.subtract(phi[0], phi[1])
 def sample_entropy(x, order=2, metric='chebyshev'):

 x = np.asarray(x, dtype=np.float64)
 if metric == 'chebyshev' and x.size < 5000:
return _numba_sampen(x, mm=order, r=0.2)
 else:
phi = _app_samp_entropy(x, order=order, metric=metric,
approximate=False)
 return -np.log(np.divide(phi[1], phi[0]))
train['SalePriceLog1p'] = np.log1p(train['SalePrice'])
 train['RealSalePriceLog1p'] = np.log1p(train['RealSalePrice'])
 print("Log1p Sale Price skeness: ",train['SalePriceLog1p'].skew())
 print("Log1p Sale Price kurtosis: ",train['SalePriceLog1p'].kurtosis())
 print("Log1p Real Sale Price skeness: ",train['RealSalePriceLog1p'].skew())
 print("Log1p Real Sale Price kurtosis: ",train['RealSalePriceLog1p'].kurtosis() )<categorify>
def entropy_and_fractal_dim(x):
return [perm_entropy(x), svd_entropy(x), app_entropy(x), sample_entropy(x), petrosian_fd(x), katz_fd(x), higuchi_fd(x)]
def get_cmap(n, name='magma'):

 return plt.cm.get_cmap(name, n )<feature_engineering>
signals = X.reshape(( len(X), X.shape[1]*X.shape[2]))
features = []
for signal in signals:
features.append(entropy_and_fractal_dim(signal))
lam = 0.15
for col in numerical_columns_skewness_train.index:
if abs(numerical_columns_skewness_train[col])> 0.75:
train[col] = boxcox1p(train[col],lam)
 for col in numerical_columns_skewness_test.index:
if abs(numerical_columns_skewness_test[col])> 0.75:
test[col] = boxcox1p(test[col],lam )<prepare_output>
features = np.array(features ).reshape(( len(features), 7))
scaler = MinMaxScaler(feature_range=(0, 1))
scaler.fit(features)
 features = scaler.transform(features )
train = train.set_index('Id')
 train.head()<prepare_output>
def model_lstm(input_shape, feat_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
feat = Input(shape=(feat_shape[1],))
bi_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True), merge_mode='concat' )(inp)
 bi_gru = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat' )(bi_lstm)
 attention = Attention(input_shape[1] )(bi_gru)
 x = concatenate([attention, feat], axis=1)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp, feat], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
test = test.set_index('Id')
 test.head()<concatenate>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_feat, train_y, val_X, val_feat, val_y = X[train_idx], features[train_idx], y[train_idx], X[val_idx], features[val_idx], y[val_idx]
model = model_lstm(train_X.shape, features.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit([train_X, train_feat], train_y, batch_size=128, epochs=50, validation_data=([val_X, val_feat], val_y), callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict([val_X, val_feat], batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
train_sale_price = train['SalePrice']
train_real_sale_price = train['RealSalePrice']
train_sale_price_log1p= train['SalePriceLog1p']
train_real_sale_price_log1p= train['RealSalePriceLog1p']
train = train.drop(['SalePrice','SalePriceLog1p','RealSalePrice','RealSalePriceLog1p'],axis=1)
 train_test = pd.concat([train,test])
 train_test.head()<categorify>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
OH_encoder = OneHotEncoder(drop='first', sparse=False)
 OH_cols = pd.DataFrame(OH_encoder.fit_transform(train_test[categorical_columns]),columns = OH_encoder.get_feature_names(categorical_columns))
OH_cols.index = train_test.index
OH_cols.head(5 )<create_dataframe>
optimal_values = threshold_search(y_val, preds_val)
 best_threshold = optimal_values['threshold']
best_score = optimal_values['matthews_correlation']
label_columns = ['Utilities','OverallQual','OverallCond','ExterQual','ExterCond','BsmtQual','BsmtExposure',
'BsmtCond','BsmtFinType1','BsmtFinType2','HeatingQC','GarageQual','KitchenQual','FireplaceQu',
'GarageCond','PoolQC']
LB_cols = pd.DataFrame(train_test[label_columns])
 LB_cols = LB_cols.replace(
 {'Utilities':{'AllPub':4,'NoSewr':3,'NoSeWa':2,'ELO':1},
'ExterQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'ExterCond':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'BsmtQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'BsmtExposure':{'Gd':4,'Av':3,'Mn':2,'No':1,'NA':0},
'BsmtCond':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'BsmtFinType1':{'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'NA':0},
'BsmtFinType2':{'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'NA':0},
'HeatingQC':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'GarageQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'KitchenQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'FireplaceQu':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'GarageCond':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'PoolQC':{'Ex':4,'Gd':3,'TA':2,'Fa':1,'NA':0}} 
)
 MinMax_Scaler = MinMaxScaler()
LB_cols = pd.DataFrame(MinMax_Scaler.fit_transform(LB_cols),columns=label_columns)
 LB_cols.index = train_test.index
LB_cols.head()<normalization>
print("Optimal Threshold : " + str(best_threshold))
print("Best Matthews Correlation : " + str(best_score))
MinMax_Scaler = MinMaxScaler()
MM_cols = pd.DataFrame(MinMax_Scaler.fit_transform(train_test[numerical_columns]),columns = [numerical_columns])
 MM_cols.index = train_test.index
MM_cols.head(5 )<merge>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
train_test_transformed = OH_cols.join(LB_cols)
 train_test_transformed = train_test_transformed.join(MM_cols)
 train_test_transformed.head(5 )<prepare_x_and_y>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
test_data_X = train_test_transformed.loc[test.index.tolist() ,:]
train_data_X = train_test_transformed.loc[train.index.tolist() ,:]<choose_model_class>
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.002631578947368421, max_iter=50, tol=0.01))
KRR = KernelRidge(alpha=0.7, coef0=10.0, degree=2, kernel='polynomial')
 ENet = make_pipeline(RobustScaler() ,ElasticNet(alpha=0.002380952380952381, l1_ratio=0.2, max_iter=150, tol=1e-05))
model_lgb = lgb.LGBMRegressor(learning_rate=0.07, n_estimators=810, num_leaves=5,objective='regression')
 rfr = RandomForestRegressor(bootstrap=False, max_depth=105, max_features='sqrt',
min_samples_leaf=3, min_samples_split=5,
n_estimators=400)
 model_xgb = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.07, max_delta_step=0, max_depth=3,
min_child_weight=1, monotone_constraints='() ',
n_estimators=450, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None )<compute_train_metric>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape
def rmsle_cv(model,X,y,n_folds=5):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X.values)
 rmse= np.sqrt(-cross_val_score(model, X.values, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<compute_test_metric>
signals = X_test_input.reshape(( len(X_test_input), X_test_input.shape[1]*X_test_input.shape[2]))
features_test = []
for signal in signals:
features_test.append(entropy_and_fractal_dim(signal))
print(f'Lasso mean RMSE:{rmsle_cv(lasso,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'Kernel Ridge mean RMSE:{rmsle_cv(KRR,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'Elastic Net mean RMSE:{rmsle_cv(ENet,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'LightGBM mean RMSE:{rmsle_cv(model_lgb,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'Random Forest mean RMSE:{rmsle_cv(rfr,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'XGBoost mean RMSE:{rmsle_cv(model_xgb,train_data_X,train_sale_price_log1p ).mean() }' )<train_model>
features_test = np.array(features_test ).reshape(( len(features_test), 7))
features_test = scaler.transform(features_test)
 np.save("features_test.npy",features_test )
class AverageStackingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_test_metric>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
averaged_models = AverageStackingModels(models =(ENet,model_lgb,model_xgb))
print(f'Averaged models mean RMSE:{rmsle_cv(averaged_models,train_data_X,train_sale_price_log1p ).mean() }' )<save_to_csv>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict([X_test_input, features_test], batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )
averaged_models.fit(train_data_X.values, train_sale_price_log1p)
 pred_test = averaged_models.predict(test_data_X.values)
 pred_test = np.expm1(pred_test)
 sample_submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 sample_submission['SalePrice'] = pred_test
sample_submission.to_csv('submission.csv',index=False )<set_options>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<load_from_csv><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()