Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<compute_test_metric>
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold,RepeatedStratifiedKFold
from keras.callbacks import *
import tensorflow as tf
weights = stack_model.coef_/stack_model.coef_.sum()
scores = [ roc_auc_score(y_train, expit(cvdata[c])) for c in cvdata.columns ]
names = [ base_models[c] for c in cvdata.columns ]
lb = [ lbscores[c] for c in cvdata.columns ]
pd.DataFrame(data={'LB score': lb, 'CV score':scores, 'weight':weights.reshape(-1)}, index=names )<compute_test_metric>
N_SPLITS = 5
sample_size = 800000
print('Stacker score: ', roc_auc_score(y_train, stack_model.predict_proba(X_train)[:,1]))<load_from_csv>
def matthews_correlation(y_true, y_pred):

 y_pred = tf.convert_to_tensor(y_pred, np.float32)
 y_true = tf.convert_to_tensor(y_true, np.float32)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
final_sub = pd.DataFrame()
subs = {m:pd.read_csv(subfiles[m] ).rename({'is_attributed':m},axis=1)for m in base_models}
first_model = list(base_models.keys())[0]
final_sub['click_id'] = subs[first_model]['click_id']<merge>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
df = subs[first_model]
for m in subs:
if m != first_model:
df = df.merge(subs[m], on='click_id')
 df.head()<predict_on_test>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
X_test = np.array(df.drop(['click_id'],axis=1 ).clip(almost_zero,almost_one ).apply(logit))
final_sub['is_attributed'] = stack_model.predict_proba(X_test)[:,1]
final_sub.head(10 )<save_to_csv>
max_num = 127
min_num = -128
final_sub.to_csv("sub_stacked.csv", index=False, float_format='%.9f' )<import_modules>
def min_max_transformation(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
import pandas as pd
import numpy as np
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from abc import abstractmethod
import lightgbm as lgb
import xgboost as xgb
from catboost import CatBoostRegressor as cat
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.svm import SVR
from sklearn.linear_model import ElasticNet, Lasso, Ridge, BayesianRidge, LassoLarsIC, LinearRegression, RANSACRegressor
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import mean_squared_log_error
from sklearn import metrics<load_from_csv>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transformation(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 features = pd.concat([train.drop(['Id', 'SalePrice'], axis=1), test.drop('Id', axis=1)])
 features['MSSubClass'] = features['MSSubClass'].apply(str )<define_variables>
def prepare_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
temporal_features = [feature for feature in features.columns if 'Yr' in feature or 'Year' in feature or 'Mo' in feature]
numeric_features = [feature for feature in features.columns if features[feature].dtype !=
'O' and feature not in temporal_features]
categorical_features = [feature for feature in features.columns if features[feature].dtype == 'O' and feature not in temporal_features]<feature_engineering>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prepare_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
def fixing_skewness(df):

 numeric_feats = df.dtypes[df.dtypes != "object"].index
skewed_feats = df[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skewed_feats[abs(skewed_feats)> 0.5]
skewed_features = high_skew.index
for feat in skewed_features:
df[feat] = boxcox1p(df[feat], boxcox_normmax(df[feat] + 1))
return df
def overfit_reducer(df):

 overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99:
overfit.append(i)
 overfit = list(overfit)
 return overfit<data_type_conversions>
print(X.shape, y.shape)
 np.save("X.npy",X)
 np.save("y.npy",y )
def create_temporal_features(features):
df = features[temporal_features].copy()
for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
pass
else:
df[feature + "_diff"] = df['YrSold'] - df[feature]
df[feature + "_diff"] = df[feature + "_diff"].fillna(df[feature + "_diff"].max())
 df['YrBltAndRemod'] = df['YearBuilt']+df['YearRemodAdd']
for feature in temporal_features:
df[feature] = df[feature].astype(str)
 df[feature] = df[feature].fillna("Missing")
 df = pd.get_dummies(df)
 df = df.drop(overfit_reducer(df), axis=1)
 return df<data_type_conversions>
df_y = pd.DataFrame(y.reshape(1,-1))
def create_categorical_features(features):
df = features[categorical_features].copy()
df['Functional'] = df['Functional'].fillna('Typ')
 df['Utilities'] = df['Utilities'].fillna('AllPub')
 df['Electrical'] = df['Electrical'].fillna("SBrkr")
 df['KitchenQual'] = df['KitchenQual'].fillna("TA")
 df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode() [0])
 df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode() [0])
 df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode() [0])
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
df[col] = df[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
for feature in categorical_features:
df[feature] = df[feature].fillna("Missing")
 df = pd.get_dummies(df)
 df = df.drop(overfit_reducer(df), axis=1)
 return df 
<feature_engineering>
tsne = TSNE(n_components=2, random_state=42, verbose = 2)
 TSNE_X = tsne.fit_transform(df )
def create_numeric_features(features):
df = features[numeric_features + ["Neighborhood"]].copy()
for col in('GarageArea', 'GarageCars'):
df[col] = df[col].fillna(0)
 for feature in numeric_features:
df[feature] = df.groupby("Neighborhood")[feature].transform(lambda x: x.fillna(x.median()))
df = df.drop("Neighborhood", axis=1)
 df['TotalHouseSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalArea'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF'] + df['GrLivArea'] + df['GarageArea']
df['Total_Bathrooms'] =(df['FullBath'] +(0.5 * df['HalfBath'])+
df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath']))
df['TotalBath'] = df['FullBath'] + 0.5 * df['HalfBath']
df['TotalLot'] = df['LotFrontage'] + df['LotArea']
df['TotalPorch'] = df['OpenPorchSF'] + df['EnclosedPorch'] + df['ScreenPorch']
df['Total_sqr_footage'] =(df['BsmtFinSF1'] + df['BsmtFinSF2'] +
df['1stFlrSF'] + df['2ndFlrSF'])
 df['TotalBsmtFin'] = df['BsmtFinSF1'] + df['BsmtFinSF2']
df['Total_porch_sf'] =(df['OpenPorchSF'] + df['3SsnPorch'] +
df['EnclosedPorch'] + df['ScreenPorch'] +
df['WoodDeckSF'])
 df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 df = df.drop(overfit_reducer(df), axis=1)
 df = fixing_skewness(df)
 return df<prepare_x_and_y>
isomap = Isomap(n_components=200)
 df_reduced = isomap.fit_transform(df )
temporal_df = create_temporal_features(features)
 categorical_df = create_categorical_features(features)
 numeric_df = create_numeric_features(features)
 all_df = pd.concat([temporal_df,categorical_df,numeric_df],axis=1)
 X_train_all = all_df[:train.shape[0]]
X_test = all_df[train.shape[0]:]
y_train_all = train["SalePrice"]<predict_on_test>
lin_pca = KernelPCA(n_components = 2, kernel="linear", fit_inverse_transform=True)
 rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.0433, fit_inverse_transform=True)
 sig_pca = KernelPCA(n_components = 2, kernel="sigmoid", gamma=0.001, coef0=1, fit_inverse_transform=True)
 plt.figure(figsize=(11, 4))
for subplot, pca, title in(( 131, lin_pca, "Linear kernel"),(132, rbf_pca, "RBF kernel, $\gamma=0.04$"),(133, sig_pca, "Sigmoid kernel, $\gamma=10^{-3}, r=1$")) :
X_reduced = pca.fit_transform(df)
 if subplot == 132:
X_reduced_rbf = X_reduced
plt.subplot(subplot)
 plt.title(title, fontsize=14)
 plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=colorlist, cmap=plt.cm.hot)
 plt.xlabel("$z_1$", fontsize=18)
 if subplot == 131:
plt.ylabel("$z_2$", fontsize=18, rotation=0)
 plt.grid(True )
class Model:
@abstractmethod
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
raise NotImplementedError 
<find_best_model_class>
lle = LocallyLinearEmbedding(n_components=2, n_neighbors=10, random_state=42)
 lle_X = lle.fit_transform(df )
class RandomForestWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
reg = make_pipeline(RobustScaler() , RandomForestRegressor(**params))
reg.fit(X_train, y_train)
 y_valid_pred = reg.predict(X_valid)
 y_pred = reg.predict(X_test)
 return y_pred, y_valid_pred, reg
class GradientBoostingRegressorWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
reg = make_pipeline(RobustScaler() , GradientBoostingRegressor(**params))
reg.fit(X_train, y_train)
 y_valid_pred = reg.predict(X_valid)
 y_pred = reg.predict(X_test)
 return y_pred, y_valid_pred, reg
class CatBoost(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
reg = make_pipeline(RobustScaler() , cat(**params))
reg.fit(X_train, y_train)
 y_valid_pred = reg.predict(X_valid)
 y_pred = reg.predict(X_test)
 return y_pred, y_valid_pred, reg
class LightGBM(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 model = lgb.train(
 params, lgb_train,
valid_sets=lgb_eval,
num_boost_round=5000,
early_stopping_rounds=100,
verbose_eval = False 
)
 y_valid_pred = model.predict(X_valid, num_iteration=model.best_iteration)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 return y_pred, y_valid_pred, model
class XGBoost(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
dtrain = xgb.DMatrix(X_train, y_train)
 deval = xgb.DMatrix(X_valid, y_valid)
 watchlist = [(deval, 'eval'),(dtrain, 'train')]
model = xgb.train(
 params, dtrain,
num_boost_round=5000,
evals=watchlist,
early_stopping_rounds=10,
verbose_eval = False 
)
 y_valid_pred = model.predict(deval)
 dtest = xgb.DMatrix(X_test)
 y_pred = model.predict(dtest)
 return y_pred, y_valid_pred, model 
<find_best_model_class>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
class KernelRidgeWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
reg = make_pipeline(RobustScaler() , KernelRidge(alpha=params['alpha'], kernel=params['kernel'], degree=params['degree'], coef0=params['coef0']))
reg.fit(X_train, y_train)
 y_valid_pred = reg.predict(X_valid)
 y_pred = reg.predict(X_test)
 return y_pred, y_valid_pred, reg
class SVRWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
reg = make_pipeline(RobustScaler() , SVR(kernel=params['kernel'], degree=params['degree'], coef0=params['coef0'], C=params['C'], epsilon=params['epsilon']))
reg.fit(X_train, y_train)
 y_valid_pred = reg.predict(X_valid)
 y_pred = reg.predict(X_test)
 return y_pred, y_valid_pred, reg<find_best_model_class>
splits = list(StratifiedKFold(n_splits=N_SPLITS,random_state=42 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
class LinearRegressionWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
lr = make_pipeline(RobustScaler() , LinearRegression())
 lr.fit(X_train, y_train)
 y_valid_pred = lr.predict(X_valid)
 y_pred = lr.predict(X_test)
 return y_pred, y_valid_pred, lr
class LassoWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
lr = make_pipeline(RobustScaler() , Lasso(alpha=params['alpha']))
lr.fit(X_train, y_train)
 y_valid_pred = lr.predict(X_valid)
 y_pred = lr.predict(X_test)
 return y_pred, y_valid_pred, lr
class RidgeWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
lr = make_pipeline(RobustScaler() , Ridge(alpha=params['alpha']))
lr.fit(X_train, y_train)
 y_valid_pred = lr.predict(X_valid)
 y_pred = lr.predict(X_test)
 return y_pred, y_valid_pred, lr
class ElasticNetWrapper(Model):
def train_and_predict(self, X_train, X_valid, y_train, y_valid, X_test, params):
lr = make_pipeline(RobustScaler() , ElasticNet(alpha=params['alpha'], l1_ratio=params['l1_ratio']))
lr.fit(X_train, y_train)
 y_valid_pred = lr.predict(X_valid)
 y_pred = lr.predict(X_test)
 return y_pred, y_valid_pred, lr 
<define_variables>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
BASE_FOLDS = 3
META_FOLDS = 3
SK_NUM = 50
SEED = [0, 1, 2, 3, 4]<init_hyperparams>
best_threshold = threshold_search(y_val, preds_val)['threshold']
config = {
"model": "stacking",
"features": [
"temporal",
"numerical_engineering",
"objects"
],
"base_models":{
"LightGBM":{
"learning_rate": 0.01,
"num_leaves": 9,
"boosting_type": "gbdt",
"colsample_bytree": 0.7270015196871651,
"reg_alpha": 0.33858409141362755,
"reg_lambda": 0.749028887833709,
"objective": "root_mean_squared_error",
"max_bin" : 55,
"bagging_fraction" : 0.8105497285545641,
"bagging_freq" : 5,
"feature_fraction" : 0.41935759813141693,
"feature_fraction_seed":9,
"bagging_seed":9,
"min_data_in_leaf" :3,
"min_sum_hessian_in_leaf" : 1
},
"XGBoost":{
"colsample_bytree":0.6959168479765205,
"gamma":0.00679351239097101,
"learning_rate":0.01,
"max_depth":3,
"min_child_weight":1.204206891928091,
"reg_lambda":0.4013441917300995,
"subsample":0.9834112317433976,
"random_state" :7,
"nthread" : -1
},
"CatBoost":{
"learning_rate": 0.01,
"iterations": 6000,
"eval_metric": "RMSE",
"random_seed": 42,
"logging_level": "Silent",
"loss_function": "RMSE",
"od_type": "Iter",
"od_wait": 100,
"one_hot_max_size": 20,
"l2_leaf_reg": 1.4264966279202298,
"depth": 5,
"rsm": 0.6253447219486281,
"random_strength": 0.009926214908500652,
"bagging_temperature": 23
},
"GradientBoosting": {
"n_estimators":6000,
"learning_rate":0.01,
"max_depth":3,
"max_features":"sqrt",
"min_samples_leaf":5,
"min_samples_split":24,
"loss":"huber",
"random_state" :5
},
"RandomForest":{
"n_estimators": 100,
"criterion": "mse",
"max_depth": 30,
"min_samples_split": 0.0020129928930074648,
"min_samples_leaf": 1,
"max_features":"sqrt",
"random_state" :7
},
"ElasticNet":{
"alpha" : 0.0005958821579546562,
"l1_ratio": 0.9497862368719193
},
"KernelRidge":{
"alpha" : 9.2304694336833,
"kernel":"polynomial",
"degree": 3,
"coef0": 7.405346770365897
},
"Lasso":{
"alpha" : 0.0006320636127160412
},
"Ridge":{
"alpha" : 21.580134483773218
},
"SVR":{
"kernel":"poly",
"degree": 2,
"coef0": 7.865345479478388,
"C": 0.09111385520064191,
"epsilon": 0.0026048466307931843
}
},
"meta_model": "CatBoost",
"params":{
"learning_rate": 0.01,
"iterations": 6000,
"eval_metric": "RMSE",
"random_seed": 42,
"logging_level": "Silent",
"loss_function": "RMSE",
"od_type": "Iter",
"od_wait": 100,
"one_hot_max_size": 20,
"l2_leaf_reg": 1.4264966279202298,
"depth": 5,
"rsm": 0.6253447219486281,
"random_strength": 0.009926214908500652,
"bagging_temperature": 23
},
"loss": "rmse",
"target_name": "SalePrice",
"ID_name": "Id"
}<compute_test_metric>
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
def evaluate_score(true, predicted, metric_name):
if metric_name == 'rmse':
return np.sqrt(metrics.mean_squared_error(true, predicted)) 
<split>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
qcut_target = pd.qcut(y_train_all, SK_NUM, labels=False)
 y_train_log = np.log(y_train_all + 1)
 base_models = config['base_models']
oof_df = pd.DataFrame(index=[i for i in range(X_train_all.shape[0])])
 y_preds_df = pd.DataFrame(index=[i for i in range(X_test.shape[0])])
 for name, params in base_models.items() :
oof = np.zeros(( X_train_all.shape[0], 1))
y_preds = []
scores = []
for seed in SEED:
kf = StratifiedKFold(n_splits=BASE_FOLDS, shuffle=True, random_state=seed)
 for train_index, valid_index in kf.split(X_train_all, qcut_target):
X_train, X_valid =(X_train_all.iloc[train_index, :], X_train_all.iloc[valid_index, :])
 y_train, y_valid =(y_train_log.iloc[train_index], y_train_log.iloc[valid_index])
 if name == "LightGBM":
model = LightGBM()
elif name == "Lasso":
model = LassoWrapper()
elif name == "Ridge":
model = RidgeWrapper()
elif name == "ElasticNet":
model = ElasticNetWrapper()
elif name == "KernelRidge":
model = KernelRidgeWrapper()
elif name == "SVR":
model = SVRWrapper()
elif name == "XGBoost":
model = XGBoost()
elif name == "RandomForest":
model = RandomForestWrapper()
elif name == "GradientBoosting":
model = GradientBoostingRegressorWrapper()
elif name == "CatBoost":
model = CatBoost()
y_pred, y_valid_pred, m = model.train_and_predict(X_train, X_valid, y_train, y_valid, X_test, params)
 oof[valid_index, :] += y_valid_pred.reshape(len(y_valid_pred), 1)/len(SEED)
 y_preds.append(y_pred)
 rmse_valid = evaluate_score(y_valid, y_valid_pred, config['loss'])
 scores.append(rmse_valid)
 score = sum(scores)/ len(scores)
 print('===CV scores===')
 print(f"\tmodel: {name}, scores: {scores}")
 print(f"\tmodel: {name}, score: {score}")
 oof_df[name] = oof
y_preds_df[name] = sum(y_preds)/ len(y_preds)
<split>
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
oof_df = pd.concat([X_train_all, oof_df], axis=1)
 y_preds_df = pd.concat([X_test, y_preds_df], axis=1)
 y_preds = []
scores = []
for seed in SEED:
kf = StratifiedKFold(n_splits=META_FOLDS, shuffle=True, random_state=seed)
 for train_index, valid_index in kf.split(X_train_all, qcut_target):
X_train, X_valid =(oof_df.iloc[train_index, :], oof_df.iloc[valid_index, :])
 y_train, y_valid =(y_train_log.iloc[train_index], y_train_log.iloc[valid_index])
 name = config['meta_model']
if name == "LightGBM":
model = LightGBM()
elif name == "Lasso":
model = LassoWrapper()
elif name == "Ridge":
model = RidgeWrapper()
elif name == "ElasticNet":
model = ElasticNetWrapper()
elif name == "KernelRidge":
model = KernelRidgeWrapper()
elif name == "SVR":
model = SVRWrapper()
elif name == "XGBoost":
model = XGBoost()
elif name == "RandomForest":
model = RandomForestWrapper()
elif name == "GradientBoosting":
model = GradientBoostingRegressorWrapper()
elif name == "CatBoost":
model = CatBoost()
y_pred, y_valid_pred, m = model.train_and_predict(X_train, X_valid, y_train, y_valid, y_preds_df, config['params'])
 y_preds.append(y_pred)
 rmse_valid = evaluate_score(y_valid, y_valid_pred, config['loss'])
 scores.append(rmse_valid)
 score = sum(scores)/ len(scores)
 print('===CV scores===')
 print(scores)
 print(score )<load_from_csv>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape
ID_name = "Id"
target_name = "SalePrice"
sub_stacking = pd.DataFrame(pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")[ID_name])
 y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.exp(y_sub)- 1
sub_stacking[target_name] = y_sub<save_to_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
sub = pd.DataFrame(pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")[ID_name])
 sub[target_name] = 0
base_subs = [
np.exp(y_preds_df["CatBoost"])- 1 ,
np.exp(y_preds_df["ElasticNet"])- 1 ,
np.exp(y_preds_df["Lasso"])- 1 ,
np.exp(y_preds_df["LightGBM"])- 1 ,
sub_stacking[target_name],
]
weight = [0.1,0.1,0.1,0.1,0.6]
for i in range(len(base_subs)) :
sub[target_name] += base_subs[i]*weight[i]
sub.to_csv(
 './submission.csv',
index=False 
)
<import_modules>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
print(sys.version )<import_modules>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<count_missing_values><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()