Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<drop_column>
RANDOM_SEED = 27
data_dir = '.. /input/vsb-power-line-fault-detection'
df_all_1 = df_all.copy()
df_all_1 = df_all.drop(['Id','SalePrice'], axis = 1 )<feature_engineering>
meta_train_df = pd.read_csv(data_dir + '/metadata_train.csv')
 meta_test_df = pd.read_csv(data_dir + '/metadata_test.csv' )
df_all_1['MissingCount'] = df_all_1.isnull().sum(axis =1)
 df_all_1['MiscFeature'] = np.where(( df_all.MiscFeature.isnull() == True), 1,0)
 df_all_1['Alley'] = np.where(( df_all.Alley.isnull() == True), 1,0)
 df_all_1['Fence'] = np.where(( df_all.Fence.isnull() == True), 1,0)
 print("There are %0.0f" %df_all_1.shape[1] +" explanatory variables left after removing varibles with more than 50 % missing data" )<drop_column>
meta_train_df.target.value_counts()
remainMiss = missingDesc.drop(['MiscFeature','Alley','Fence','SalePrice'], axis =0)
 df_all_1[remainMiss.index].dtypes<feature_engineering>
meta_train_df.id_measurement.nunique()
PoolIndex = df_all_1[df_all_1['PoolArea']!=0].index
df_all_1.loc[np.logical_not(df_all_1.index.isin(PoolIndex)) ,'PoolQC'] = 'NA'
df_all_1.loc[PoolIndex,'PoolQC'].isnull().sum()
df_all_1['PoolQC'] = df_all_1['PoolQC'].fillna(df_all_1.PoolQC[PoolIndex].mode() [0])
 df_all_1.PoolQC.value_counts()<feature_engineering>
meta_train_df.groupby('id_measurement' ).target.sum().value_counts()
FireplaceIndex = df_all_1[df_all_1['Fireplaces']==0].index
df_all_1.loc[df_all_1.index.isin(FireplaceIndex),'FireplaceQu'] = 'NA'
df_all_1.loc[FireplaceIndex,'PoolQC'].isnull().sum() 
<count_unique_values>
train_df = pd.read_parquet(data_dir + '/train.parquet' )
cat_length = 30
print('Imputing non-numeric missing variables...')
 df_all_1 = df_all_1.apply(lambda x: x.fillna(x.mode() [0])if(x.dtype == "object" and len(x.unique())<= cat_length)else x, axis=0 )<sort_values>
negative_signal_ids = meta_train_df[meta_train_df.id_measurement==1287].signal_id.values
positive_signal_ids = meta_train_df[meta_train_df.id_measurement==2649].signal_id.values
negative_sample = train_df.iloc[:, negative_signal_ids].values
positive_sample = train_df.iloc[:, positive_signal_ids].values
pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )<count_values>
assert numba.__version__ == '0.46.0'
for col in remainMiss.index:
if(df_all_1[col].dtypes != 'object' and len(np.unique(df_all_1[col])) < cat_length):
print(df_all_1[col].value_counts() )<feature_engineering>
@numba.jit(nopython=True)
 def ema_residuals(x, alpha=0.01):

 new_x = np.zeros_like(x)
 ema = x[0]
for i in range(1, len(x)) :
ema = ema*(1-alpha)+ alpha*x[i]
new_x[i] = x[i] - ema
return new_x
print('Imputing 3 missing numeric variables as categorical varaibles...')
 for col in ['BsmtHalfBath', 'BsmtFullBath','GarageCars']:
df_all_1[col] = df_all_1[col].fillna(df_all_1[col].mode() [0])
 pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )<feature_engineering>
@numba.jit(nopython=True)
 def ema(x, alpha=0.01):

 new_x = np.zeros_like(x)
 ema = x[0]
for i in range(1, len(x)) :
ema = ema*(1-alpha)+ alpha*x[i]
new_x[i] = ema
return new_x
print("Imputing GarageYrBlt...")
 temp = df_all_1[df_all_1.GarageYrBlt.isnull() == True].index
df_all_1["GarageYrBlt"]= np.where(( df_all_1.GarageYrBlt.isnull() == True), df_all_1.YearBuilt,df_all_1.GarageYrBlt ).astype('int64')
 print('Chekcing if GarageYrBlt is corretly imputed...')
(df_all_1.loc[temp,"GarageYrBlt"] == df_all_1.loc[temp,"YearBuilt"] ).unique()<create_dataframe>
flat_negative_sample = np.zeros_like(negative_sample)
 flat_positive_sample = np.zeros_like(positive_sample)
 for i in range(3):
flat_negative_sample[:,i] = ema_residuals(negative_sample[:,i])
 flat_positive_sample[:,i] = ema_residuals(positive_sample[:,i] )
imputeCol = []
for col in df_all_1.columns:
if(df_all_1[col].dtypes != 'object' and len(df_all_1[col].unique())>= cat_length):
imputeCol.append(col)
 print("Columns used for KNN imputation are:
" + str(imputeCol))
imputer = KNNImputer(n_neighbors=5, weights='uniform', metric='nan_euclidean')
 X = df_all_1[imputeCol]
print('Imputing remaining numerical variables with KNN...')
 Xtrans = imputer.fit_transform(X)
 XT = pd.DataFrame(data=Xtrans)
 XT.columns = imputeCol
df_all_1[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']] \
= XT[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']]
print('There are now ' + str(df_all_1.isnull().any().sum())+ ' missing values in our data set.' )<sort_values>
flat_negative_sample = np.zeros_like(negative_sample)
 flat_positive_sample = np.zeros_like(positive_sample)
 for i in range(3):
flat_negative_sample[:,i] = ema(negative_sample[:,i])
 flat_positive_sample[:,i] = ema(positive_sample[:,i] )
numVal =(( df_all_1.dtypes != 'object' ).index)[df_all_1.dtypes != 'object']
length = []
for col in numVal:
length.append(len(df_all_1[col].unique()))
numVal_df = pd.DataFrame({'Variables':numVal,'UniqueValuesLength':length})
 num_cat_Val = numVal_df[numVal_df['UniqueValuesLength'] < cat_length].sort_values(by='UniqueValuesLength', ascending=False)
 num_cat_Val.head()<filter>
@numba.jit(nopython=True)
 def drop_missing(intersect,sample):

 i=j=k=0
new_intersect=np.empty_like(intersect)
 while i< intersect.size and j < sample.size:
if intersect[i]==sample[j]:
new_intersect[k]=intersect[i]
k+=1
i+=1
j+=1
elif intersect[i]<sample[j]:
i+=1
else :
j+=1
return new_intersect[:k]
@numba.jit(nopython=True)
 def _local_maxima_1d_window_single_pass(x, w):
midpoints = np.empty(x.shape[0] // 2, dtype=np.intp)
 left_edges = np.empty(x.shape[0] // 2, dtype=np.intp)
 right_edges = np.empty(x.shape[0] // 2, dtype=np.intp)
 m = 0
i = 1
i_max = x.shape[0] - 1
while i < i_max:
if x[i - 1] < x[i]:
i_ahead = i + 1
while i_ahead < i_max and x[i_ahead] == x[i]:
i_ahead += 1
i_right = i_ahead - 1
f = False
i_window_end = i_right + w
while i_ahead < i_max and i_ahead < i_window_end:
if x[i_ahead] > x[i]:
f = True
break
i_ahead += 1
if x[i_ahead] < x[i]:
left_edges[m] = i
right_edges[m] = i_right
midpoints[m] =(left_edges[m] + right_edges[m])// 2
m += 1
i = i_ahead - 1
i += 1
midpoints = midpoints[:m]
left_edges = left_edges[:m]
right_edges = right_edges[:m]
return midpoints, left_edges, right_edges
@numba.jit(nopython=True)
 def local_maxima_1d_window(x, w=1):

 fm, fl, fr = _local_maxima_1d_window_single_pass(x, w)
 bm, bl, br = _local_maxima_1d_window_single_pass(x[::-1], w)
 bm = np.abs(bm - x.shape[0] + 1)[::-1]
bl = np.abs(bl - x.shape[0] + 1)[::-1]
br = np.abs(br - x.shape[0] + 1)[::-1]
m = drop_missing(fm, bm)
 return m
print('There are %i' %numVal_df[numVal_df['UniqueValuesLength'] < cat_length].shape[0] + ' numerical variables and %i' \
%(df_all_1.dtypes == 'object' ).sum() + ' non-numerical variables can potentially be treated as categorcial variables with some exceptions.' )<sort_values>
def get_peaks(
 x,
window=25,
visualise=False,
visualise_color=None, 
):

 x_flatten = ema_residuals(x)
 x_flatten_abs = np.abs(x_flatten)
 peaks_indices = local_maxima_1d_window(x_flatten_abs, window)
 heights = x_flatten_abs[peaks_indices]
peaks_sorted_indices = np.argsort(heights)[::-1]
peaks_indices = peaks_indices[peaks_sorted_indices]
heights = heights[peaks_sorted_indices]
ky = heights
kx = np.arange(1, heights.shape[0]+1)
 conv_length = 9
grad = np.diff(ky, 1)/np.diff(kx, 1)
 grad = np.convolve(grad, np.ones(conv_length)/conv_length)
 grad = grad[conv_length-1:-conv_length+1]
knee_x = plateau_detection(grad, -0.01, plateau_length=1000)
 knee_x -= conv_length//2
if visualise:
plt.plot(grad, color=visualise_color)
 plt.axvline(knee_x, ls="--", color=visualise_color)
 peaks_x = peaks_indices[:knee_x]
peaks_y = heights[:knee_x]
ii = np.argsort(peaks_x)
 peaks_x = peaks_x[ii]
peaks_y = peaks_y[ii]
return peaks_x, peaks_y
@numba.jit(nopython=True)
 def plateau_detection(grad, threshold, plateau_length=5):

 count = 0
loc = 0
for i in range(grad.shape[0]):
if grad[i] > threshold:
count += 1
if count == plateau_length:
loc = i - plateau_length
break
return loc
cont_val = numVal_df[numVal_df['UniqueValuesLength'] >= cat_length].sort_values(by='UniqueValuesLength')
 cont_val.head()<sort_values>
@numba.jit(nopython=True, parallel=True)
 def calculate_current_phase(data):

 n = 800000
assert data.shape[0] == n
omegas = np.exp(-2j * np.pi * np.arange(n)/ n)
 res = np.zeros(data.shape[1], dtype=omegas.dtype)
 for i in numba.prange(data.shape[1]):
res[i] = omegas.dot(data[:, i].astype(omegas.dtype))
return np.angle(res, deg=False)
 def to_angle(x, phase):
dt = 1/800000
return(np.degrees(2*np.pi*dt*x + phase)+ 90)% 360
yCor = cor['SalePrice'].sort_values(ascending = False ).reset_index().drop(index=0)
 yCor.columns = ["Variables", "Cor"]
yCor[(yCor["Cor"] > 0.5)|(yCor["Cor"] < -0.5)]<create_dataframe>
def process_measurement(data_df, meta_df):

 peaks = []
for i, sig_id in enumerate(meta_df.signal_id):
mat = []
signal = data_df.iloc[:, i].values.astype(np.float)
 px, h = get_peaks(signal)
 mat.append(px)
 mat.append(h)
 mat.append([sig_id]*len(px))
peaks.append(np.asarray(mat))
peaks = pd.DataFrame(
 np.concatenate(peaks, axis=1 ).T,
columns=['px', 'height', 'signal_id'] 
)
 phase_50hz = calculate_current_phase(data_df.values)
 phase_50hz = pd.DataFrame(
 phase_50hz,
columns=['phase_50hz'] 
)
 phase_50hz['signal_id'] = meta_df['signal_id'].values
peaks = pd.merge(peaks, phase_50hz, on='signal_id', how='left')
 peaks['phase_aligned_x'] = to_angle(peaks['px'], peaks['phase_50hz'])
 peaks['Q'] = pd.cut(peaks['phase_aligned_x'], [0, 90, 180, 270, 360], labels=[0, 1, 2, 3])
 return peaks
df_all_red = df_all_1.copy()<feature_engineering>
train_peaks = process_measurement(train_df, meta_train_df)
 train_peaks = pd.merge(train_peaks, meta_train_df[['signal_id', 'id_measurement', 'target']], on='signal_id', how='left')
 del train_df
gc.collect()
df_all_red.YearRemodAdd = np.where(df_all_1.YearBuilt >df_all_1.YearRemodAdd, df_all_1.YearBuilt, df_all_1.YearRemodAdd)
 df_all_red.YrSold = np.where(df_all_1.YearBuilt >df_all_1.YrSold , df_all_1.YearBuilt, df_all_1.YrSold)
 df_all_red.YrSold = np.where(df_all_red.YearRemodAdd > df_all_red.YrSold, df_all_1.YearRemodAdd, df_all_red.YrSold)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt < df_all_red.YearBuilt, df_all_red.YearBuilt, df_all_red.GarageYrBlt)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt >2010, df_all_red.YearBuilt,df_all_red.GarageYrBlt)
 df_all_red['RestorePeriod'] = df_all_red.YearRemodAdd - df_all_red.YearBuilt
df_all_red['HouseAge'] = df_all_red.YrSold - df_all_red.YearBuilt
df_all_red["RemodAge"] = df_all_red.YrSold - df_all_red.YearRemodAdd
df_all_red['GarageAge'] = df_all_red.YrSold - df_all_red.GarageYrBlt
df_all_red["IsNewHouse"] = np.where(df_all_red.YearBuilt == df_all_red.YrSold, 1 ,0)
 df_all_red["Remodeled"] = np.where(df_all_red.RestorePeriod != 0, 1 ,0)
 df_all_red["GreatDepression"] = np.where(( df_all_1.YearBuilt >=1929)&(df_all_1.YearBuilt <= 1939), 1,0)
 df_all_red["Pre-GreatDepression"] = np.where(( df_all_1.YearBuilt <1929), 1,0)
 df_all_red["MortgageCrisis"] = np.where(( df_all_1.YearBuilt >=2007)&(df_all_1.YearBuilt <= 2010), 1,0)
 df_all_red["MortgageCrisisRestoration"] = np.where(( df_all_1.YearRemodAdd >=2007)&(df_all_1.YearRemodAdd <= 2010), 1,0)
<drop_column>
def calculate_features(peaks_df, meta_df):
results = pd.DataFrame(index=meta_df['id_measurement'].unique())
 results.index.rename('id_measurement', inplace=True)
 res = peaks_df.groupby('id_measurement' ).agg({
'px': 'count',
})
 res.columns = ["peak_count_total"]
results = pd.merge(results, res, on='id_measurement', how='left')
 p = peaks_df[peaks_df['Q'].isin([0, 2])].copy()
res = p.groupby('id_measurement' ).agg({
'px': 'count',
})
 res.columns = ["peak_count_Q02"]
results = pd.merge(results, res, on='id_measurement', how='left')
 p = peaks_df[peaks_df['Q'].isin([1, 3])].copy()
res = p.groupby('id_measurement' ).agg({
'px': 'count',
})
 res.columns = ['peak_count_Q13']
results = pd.merge(results, res, on='id_measurement', how='left')
 p = peaks_df[peaks_df['Q'].isin([0, 2])].copy()
res = p.groupby('id_measurement' ).agg({
'height': ['mean', 'std'],
})
 res.columns = ["_".join(f)+ '_Q02' for f in res.columns]
results = pd.merge(results, res, on='id_measurement', how='left')
 return results
df_all_red.drop(["YearBuilt","GarageYrBlt", "YearRemodAdd"], axis = 1, inplace = True )<feature_engineering>
X_train = calculate_features(train_peaks, meta_train_df )
df_all_red["TotalFlrSF"] = df_all_red["1stFlrSF"] + df_all_red["2ndFlrSF"]
df_all_red["TotalBsmtFin"] = df_all_red['BsmtFinSF1']+df_all_red['BsmtFinSF2']
df_all_red["BsmtFinRatio"] =(df_all_red['TotalBsmtFin'] / df_all_red['TotalBsmtSF'] ).fillna(0)
 df_all_red["IsBasement"] = np.where(df_all_red['TotalBsmtSF'] < 1 , 0, 1)
 df_all_red["PorchArea"] = df_all_red['OpenPorchSF'] + df_all_red['EnclosedPorch'] + df_all_red['ScreenPorch'] + \
df_all_red["3SsnPorch"] + df_all_red['WoodDeckSF']
df_all_red["TotalHouseArea"] = df_all_red['TotalFlrSF'] + df_all_red['TotalBsmtSF']
df_all_red["TotalLivArea"] = df_all_red['GrLivArea'] + df_all_red['TotalBsmtSF']
df_all_red["2ndFloor"] = np.where(( df_all_red['2ndFlrSF'] < 1), 0, 1)
 df_all_red["GarageAreaPerCar"] =(df_all_red.GarageArea / df_all_red.GarageCars ).fillna(0)
 df_all_red["HouseLotRatio"] = df_all_red.TotalHouseArea /df_all_red.LotArea
df_all_red["LivLotRatio"] = df_all_red.GrLivArea / df_all_red.LotArea
df_all_red["GarageLotRatio"] = df_all_red.GarageArea / df_all_red.LotArea
df_all_red["MasVnrRatio"] = df_all_red.MasVnrArea / df_all_red.TotalHouseArea
df_all_red["DifArea"] =(df_all_red.LotArea - df_all_red["1stFlrSF"] - df_all_red.GarageArea - df_all_red.PorchArea)
 df_all_red["LowQualFinSFRatio"] = df_all_red.LowQualFinSF/df_all_red.TotalHouseArea<count_values>
y_train = meta_train_df.groupby('id_measurement')['target'].sum() > 0
y_train = y_train.astype(np.float )
for col in ['TotRmsAbvGrd', 'OverallQual', 'OverallCond','BedroomAbvGr','GarageCars','KitchenAbvGr','FullBath','MissingCount']:
print(col + ' has ' + str(len(df_all_red[col].value_counts())) + ' unique numerical values.' )<feature_engineering>
num_folds = 5
np.random.seed(13)
 splits = np.zeros(X_train.shape[0], dtype=np.int)
 m = y_train == 1
splits[m] = np.random.randint(0, num_folds, size=m.sum())
 m = y_train == 0
splits[m] = np.random.randint(0, num_folds, size=m.sum() )
for col in ['OverallQual','OverallCond']:
df_all_red[col].replace(np.linspace(1,10,10 ).astype(int), list(np.linspace(10,100,10)) , inplace=True)
 df_all_red['OverallScore'] = df_all_red[['OverallQual', 'OverallCond']].sum(axis = 1)/2
df_all_red["OverallGrade"] = df_all_red["OverallQual"] * df_all_red["OverallCond"]<categorify>
pd.Series(splits ).value_counts()
for col in ['BsmtFinType1','BsmtFinType2']:
df_all_red[col].replace(to_replace=['Unf', 'LwQ','Rec','BLQ','ALQ','GLQ'], value=[1/6,2/6,3/6,4/6,5/6,1], inplace=True)
 for col in ['BsmtCond', 'BsmtQual', 'GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual','HeatingQC','FireplaceQu','PoolQC']:
df_all_red[col].replace(to_replace=['NA','Po','Fa', 'TA','Gd','Ex'], value=[0,1/5,2/5,3/5,4/5,1], inplace=True)
 df_all_red["Functional"].replace(['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],[1/7,2/7,3/7,4/7,5/7,6/7,1] , inplace = True)
 df_all_red["BsmtExposure"].replace(['No','Mn','Av','Gd'],[1/4,2/4,3/4,1] , inplace = True)
 ratingCols= ['BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','BsmtExposure','GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual', \
"HeatingQC","Functional",'FireplaceQu','PoolQC']
df_all_red[ratingCols] = df_all_red[ratingCols]*100
df_all_red['BsmtScore']= df_all_red[["BsmtCond", "BsmtQual","BsmtExposure","BsmtFinType1", "BsmtFinType2"]].sum(axis = 1)/5
df_all_red['GarageScore']=df_all_red[["GarageQual", "GarageCond"]].sum(axis = 1)/2
df_all_red['ExterScore'] = df_all_red[["ExterQual", "ExterCond"]].sum(axis = 1)/2
df_all_red['QualScore'] = df_all_red[["OverallQual", "ExterQual", "GarageQual", "BsmtQual","BsmtFinType1", "BsmtFinType2", "KitchenQual","PoolQC", \
"FireplaceQu"]].sum(axis = 1)/9
df_all_red['CondScore'] = df_all_red[["OverallCond", "ExterCond", "GarageCond", "BsmtCond", "HeatingQC", "Functional"]].sum(axis = 1)/6
df_all_red['GarageGrade'] = df_all_red["GarageQual"] * df_all_red["GarageCond"]
df_all_red['ExterGrade'] = df_all_red["ExterQual"] * df_all_red["ExterCond"]
df_all_red["KitchenGrade"] = df_all_red["KitchenAbvGr"] * df_all_red["KitchenQual"]
df_all_red['BsmtGrade'] = df_all_red["BsmtCond"] * df_all_red["BsmtQual"]
df_all_red['FireplaceGrade'] = df_all_red["Fireplaces"] * df_all_red["FireplaceQu"]
df_all_red['PoolGrade'] = df_all_red["PoolArea"] * df_all_red["PoolQC"] 
<count_values>
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 80,
'num_boost_round': 10000,
'feature_fraction': 0.8,
'bagging_fraction': 0.8,
'num_threads': 4,
'seed': 23974,
}
for col in num_cat_Val['Variables']:
print(df_all_red[col].value_counts() )<feature_engineering>
models = []
cv_scores = []
val_cv_scores = []
feature_names = X_train.columns.tolist()
yp_train = np.zeros(X_train.shape[0])
 yp_val = np.zeros(X_train.shape[0])
 yp_test = np.zeros(X_train.shape[0])
 for fold in range(num_folds):
val_fold = fold
test_fold =(fold + 1)% num_folds
train_folds = [f for f in range(num_folds)if f not in [val_fold, test_fold]]
train_indices = np.where(np.isin(splits, train_folds)) [0]
val_indices = np.where(splits == val_fold)[0]
test_indices = np.where(splits == test_fold)[0]
trn = lgb.Dataset(
 X_train.values[train_indices],
y_train[train_indices],
feature_name=feature_names, 
)
 val = lgb.Dataset(
 X_train.values[val_indices],
y_train[val_indices],
feature_name=feature_names, 
)
 test = lgb.Dataset(
 X_train.values[test_indices],
y_train[test_indices],
feature_name=feature_names, 
)
 model = lgb.train(
 params,
trn,
valid_sets=(trn, test, val),
valid_names=("train", "test", "validation"),
early_stopping_rounds=10,
verbose_eval=50 
)
 yp = model.predict(X_train.values[train_indices])
 yp_train[train_indices] += yp
yp_val_fold = model.predict(X_train.values[val_indices])
 yp_val[val_indices] += yp_val_fold
yp_test_fold = model.predict(X_train.values[test_indices])
 yp_test[test_indices] += yp_test_fold
models.append(model)
 cv_scores.append(model.best_score['test']['binary_logloss'])
 val_cv_scores.append(model.best_score['validation']['binary_logloss'])
 yp_train /=(num_folds - 2)
 cv_scores = np.asarray(cv_scores)
 val_cv_scores = np.asarray(val_cv_scores )
df_all_red["MSSubClass_1-Story"] = np.where(df_all_red.MSSubClass.isin([20,30,40,120]), 1, 0)
 df_all_red["MSSubClass_1-1/2Story"] = np.where(df_all_red.MSSubClass.isin([45,50,150]), 1, 0)
 df_all_red["MSSubClass_2-Story"] =np.where(df_all_red.MSSubClass.isin([60,70,160]), 1, 0)
 df_all_red["MSSubClass_2-1/2Story"] = np.where(df_all_red.MSSubClass.isin([75]), 1, 0)
 df_all_red["MSSubClass_Split"] = np.where(df_all_red.MSSubClass.isin([80,85]), 1, 0)
 df_all_red["MSSubClass_AllAges"] = np.where(df_all_red.MSSubClass.isin([40,45,50,75,90,150, 190]), 1, 0)
 df_all_red["MSSubClass_Newer"] = np.where(df_all_red.MSSubClass.isin([20, 60, 120, 160]), 1, 0)
 df_all_red["MSSubClass_Older"] = np.where(df_all_red.MSSubClass.isin([30, 70]), 1, 0)
 df_all_red["MSSubClass_AllStyles"] = np.where(df_all_red.MSSubClass.isin([20, 90, 190]), 1, 0)
 df_all_red["MSSubClass_Pud"] = np.where(df_all_red.MSSubClass.isin([150, 160, 180]), 1, 0)
 df_all_red["MSSubClass_SplitMultiLevel"] = np.where(df_all_red.MSSubClass.isin([80,85, 180]), 1, 0)
<drop_column>
print("CV Val Logloss: {:.4f} +/- {:.4f}({:.4f})".format(val_cv_scores.mean() , val_cv_scores.std() /np.sqrt(val_cv_scores.shape[0]), val_cv_scores.std()))
print("CV Test Logloss: {:.4f} +/- {:.4f}({:.4f})".format(cv_scores.mean() , cv_scores.std() /np.sqrt(cv_scores.shape[0]), cv_scores.std()))
print("Train accuracy: {:.4f}".format(metrics.accuracy_score(y_train, yp_train > 0.5)))
 print("CV Val accuracy: {:.4f}".format(metrics.accuracy_score(y_train, yp_val > 0.5)))
 print("CV Test accuracy: {:.4f}".format(metrics.accuracy_score(y_train, yp_test > 0.5)) )
df_all_red.drop(["MSSubClass"], axis = 1, inplace = True )<feature_engineering>
thresholds = np.linspace (.01,.9, 90)
 scores_train = []
scores_val = []
scores_test = []
for t in thresholds:
s_train = metrics.f1_score(
 y_train.values.astype(np.float),
yp_train > t 
)
 s_val = metrics.f1_score(
 y_train.values.astype(np.float),
yp_val > t 
)
 s_test = metrics.f1_score(
 y_train.values.astype(np.float),
yp_test > t 
)
 scores_train.append(s_train)
 scores_val.append(s_val)
 scores_test.append(s_test)
 plt.plot(thresholds, scores_train)
 plt.plot(thresholds, scores_val)
 plt.plot(thresholds, scores_test)
 plt.axvline(thresholds[np.argmax(scores_val)], ls='--')
 plt.xlabel('Threshold')
 plt.ylabel('F1 Score')
 plt.show()
print(round(np.max(scores_val), 4), thresholds[np.argmax(scores_val)])
 best_thresh = thresholds[np.argmax(scores_val)]
print("Collapsing categories...")
 df_all_red['LotShape'] = np.where(df_all_1.LotShape != 'Reg', 'IR', df_all_1['LotShape'])
 df_all_red['LandContour'] = np.where(df_all_1.LandContour != 'Lvl', 'IR', df_all_1['LandContour'])
 df_all_red['RoofMatl'] = np.where(df_all_1.RoofMatl != 'CompShg', 'Non-Standard', df_all_1['RoofMatl'])
 df_all_red['Heating'] = np.where(df_all_1.Heating.str.contains("Gas"), 1, 0)
 df_all_red["MSZoning"] = np.where(df_all_red.MSZoning.isin(["RH", "RM", "RL"]), 1, 0)
 df_all_red['Electrical'] = np.where(df_all_1.Electrical != 'SBrkr', 'non-Standard', df_all_1['Electrical'])
 df_all_red['SaleCondition'] = np.where(df_all_1.SaleCondition != 'Normal', 'non-Normal', df_all_1['SaleCondition'])
 df_all_red["SaleType"] = np.where(df_all_1.SaleType.isin(["Con", "ConLD", "ConLI", "ConLw"]), "Contract", df_all_1.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["WD", "CWD"]), "Warranty", df_all_red.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["COD"]), "Oth", df_all_red.SaleType)
 df_all_red["Foundation"] = np.where(df_all_1["Foundation"] == "Stone", "BrkTil", df_all_1["Foundation"])
 df_all_red["Foundation"] = np.where(df_all_red["Foundation"] == "Wood", "PConc", df_all_red["Foundation"])
 df_all_red["BldgType"] = np.where(df_all_1.BldgType.isin(['1Fam', '2fmCon']), "Family", df_all_1.BldgType)
 df_all_red["BldgType"] = np.where(df_all_red.BldgType.isin(['TwnhsE', 'Twnhs']), "Townhouse", df_all_red.BldgType )<feature_engineering>
best_thresh = 0.4
df_all_red["CNormal"] = np.where(( df_all_1.Condition1.isin(["Norm"])) |(df_all_1.Condition2.isin(["Norm"])) , 1, 0)
 df_all_red["CRailroad"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) , 1, 0)
 df_all_red["CPark"] = np.where(( df_all_1.Condition1.isin(["PosN", "PosA"])) |(df_all_1.Condition2.isin(["PosN", "PosA"])) , 1, 0)
 df_all_red["CAdjacent"] = np.where(( df_all_1.Condition1.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) |(df_all_1.Condition2.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) , 1, 0)
 df_all_red["CWithin"] = np.where(( df_all_1.Condition1.isin(["RRNn", "PosN", "RRNe"])) |(df_all_1.Condition2.isin(["RRNn", "PosN", "RRNe"])) , 1, 0)
 df_all_red["CNorthS"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn"])) , 1, 0)
 df_all_red["CEastW"] = np.where(( df_all_1.Condition1.isin(["RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNe", "RRAe"])) , 1, 0)
 df_all_red["CStreet"] = np.where(( df_all_1.Condition1.isin(["Artery", "Feedr"])) |(df_all_1.Condition2.isin(["Artery", "Feedr"])) , 1, 0)
 df_all_red["HouseStyle_Floor"] = np.where(df_all_1.HouseStyle.isin(["1Story"]), "1", "2")
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["1.5Fin", "1.5Unf"]), "1.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["2.5Fin", "2.5Unf"]), "2.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["SLvl", "SFoyer"]), "Split", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_IsSplit"] = np.where(df_all_red.HouseStyle_Floor == 'Split',1,0)
 df_all_red['HasMasVnrType'] = np.where(df_all_1['MasVnrType'] == "None",0,1 )<drop_column>
pred_problem = yp_test > best_thresh
pred_neg = yp_test <= best_thresh
true_problem = y_train > 0.5
true_neg = y_train <= 0.5
df_all_red.drop(["Condition1", "Condition2", "YrSold"], axis = 1, inplace = True )<create_dataframe>
del X_train, train_peaks
gc.collect()
df_all_2 = df_all_red.copy()<categorify>
NUM_TEST_CHUNKS = 10
test_chunk_size = int(np.ceil(( meta_test_df.shape[0]/3.) /float(NUM_TEST_CHUNKS)) *3.)
test_peaks = []
for j in range(NUM_TEST_CHUNKS):
j_start = j*test_chunk_size
j_end =(j+1)*test_chunk_size
signal_ids = meta_test_df['signal_id'].values[j_start:j_end]
test_df = pd.read_parquet(
 data_dir + '/test.parquet',
columns=[str(c)for c in signal_ids] 
)
 p = process_measurement(
 test_df,
meta_test_df.iloc[j_start:j_end], 
)
 test_peaks.append(p)
 print(j)
 del test_df
gc.collect()
test_peaks = pd.concat(test_peaks )
labelencoder = preprocessing.LabelEncoder()
label_cols = [col for col in df_all_red.columns if len(df_all_red[col].value_counts())== 2]
for col in label_cols:
df_all_2[col] = labelencoder.fit_transform(df_all_red[col] )<define_variables>
test_peaks = pd.merge(test_peaks, meta_test_df[['signal_id', 'id_measurement']], on='signal_id', how='left')
 test_peaks.head()
cat_cols = df_all_2.columns[(df_all_2.dtypes== 'object')|(df_all_2.dtypes =='category')]
print("There are " + str(len(cat_cols)) +" categorical variables require one hot encoding.")
 cat_cols<categorify>
X_test = calculate_features(test_peaks, meta_test_df )
print("One hot encoding...")
 original_columns = list(df_all_2.columns)
 df_final= pd.get_dummies(df_all_2, columns=cat_cols, dummy_na=False, drop_first=True)
 new_columns = [c for c in df_final.columns if c not in original_columns]
print("There are " + str(len(new_columns)) + " variables have been created after one hot encoding" )<save_to_csv>
yp_test = np.zeros(X_test.shape[0])
 for j in range(len(models)) :
model = models[j]
yp_test += model.predict(X_test.values)/len(models )
<load_from_csv>
test_submission = pd.DataFrame(
 yp_test,
index=X_test.index,
columns=['probability'] 
)
 test_submission['target'] =(yp_test > best_thresh ).astype(np.int)
 test_submission = pd.merge(
 meta_test_df[['id_measurement', 'signal_id']],
test_submission,
on='id_measurement',
how='left'
)
<feature_engineering>
test_submission.target.value_counts()
<prepare_x_and_y><EOS> 
submission = test_submission[['signal_id', 'target']]
submission.to_csv('submission.csv', index=False )