Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<create_dataframe>

 pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
pd.set_option("display.max_columns", 81)
 pd.set_option("display.max_rows", 101)
 pd.set_option("display.max_colwidth", 100)

plt.style.use('bmh')
 sns.set_style({'axes.grid':False})
 sns.set_style('darkgrid')
 %matplotlib inline


wrn.filterwarnings('ignore', category = DeprecationWarning)
 wrn.filterwarnings('ignore', category = FutureWarning)
 wrn.filterwarnings('ignore', category = UserWarning )
X_rg = pd.DataFrame()
for col in features.columns:
rg = rank_gauss(features[col])
 X_rg = pd.concat([X_rg, rg], axis=1)
 X_rg<prepare_x_and_y>

 train_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
X = X_rg.iloc[:len(train_features), :]
X_test = X_rg.iloc[len(train_features):, :]
X = np.array(X)
 X_test = np.array(X_test )<load_from_csv>

 train_Id = train_df['Id']
test_Id = test_df['Id']
train = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/train.csv")
 y = train["time_to_eruption"]
y<import_modules>

 all_df = pd.concat(( train_df, test_df)).reset_index(drop=True)
 all_df.drop(['SalePrice'], axis=1, inplace=True)
 print("The all data size is : {}".format(all_df.shape))
import lightgbm as lgb
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error as mae
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials<split>

 kf = KFold(n_splits=5, shuffle=True, random_state=2)

def rmsle_cv(model, X, y, cv=kf):
rmsle = np.sqrt(
 -cross_val_score(
 model,
X, y,
scoring="neg_mean_squared_log_error",
cv=cv, 
)
)
 return(rmsle )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=34)
 print(X_train.shape)
 print(X_val.shape )<choose_model_class>

 na_count_all_df = all_df.isnull().sum()
na_count_all_df = na_count_all_df.drop(na_count_all_df[na_count_all_df == 0].index)
 na_ratio_all_df = na_count_all_df / len(all_df)
 missing_data = pd.DataFrame({
'All Missing Ratio': na_ratio_all_df,
'All Missing Count': na_count_all_df,
'Train Missing Count': train_df[na_count_all_df.index].isnull().sum() ,
'Test Missing Count': test_df[na_count_all_df.index].isnull().sum() ,
'All Data Type': all_df[na_count_all_df.index].dtypes,
'Train Data Type': train_df[na_count_all_df.index].dtypes,
'Test Data Type': test_df[na_count_all_df.index].dtypes,
} ).sort_values('All Missing Count', ascending=False)
 missing_data.head(100 )
kf1 = KFold(n_splits=10, shuffle=True, random_state=95)
 kf2 = KFold(n_splits=10, shuffle=True, random_state=43 )<create_dataframe>
all_df['PoolQC'] = all_df['PoolQC'].fillna('None' )
lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_val, label=y_val, reference=lgb_train )<train_model>
all_df['MiscFeature'] = all_df['MiscFeature'].fillna('None' )
def lgb_score(params):
params["num_leaves"] = int(params["num_leaves"])
 params["min_data_in_leaf"] = int(params["min_data_in_leaf"])
 model = lgb.train(params=lgb_params,
train_set=lgb_train,
valid_sets=(lgb_train, lgb_val),
num_boost_round=10000,
early_stopping_rounds=20,
verbose_eval=0)
 pred = model.predict(X_val, num_iteration=model.best_iteration)
 score = mae(pred, y_val)
 history.append(( params, score))
return {"loss": score, "status": STATUS_OK}<init_hyperparams>
all_df['Alley'] = all_df['Alley'].fillna('None' )
lgb_params = {
"task": "train",
"boosting_type": "gbdt",
"objective": "regression",
"eval_metric": "mae",
"num_leaves": 31,
"learning_rate": 0.08,
"bagging_fraction": 0.8,
"feature_fraction": 0.8,
"min_data_in_leaf": 20,
"random_state": 93
}
params_space = {
"num_leaves": hp.quniform("num_leaves", 25, 50, 1),
"min_data_in_leaf": hp.quniform("min_data_in_leaf", 10, 70, 1),
"bagging_fraction": hp.quniform("bagging_fraction", 0.6, 0.95, 0.025),
"feature_fraction": hp.quniform("feature_fraction", 0.6, 0.95, 0.025)
 }
trials = Trials()
history = []
fmin(lgb_score, params_space, algo=tpe.suggest, trials=trials, max_evals=100)
 history = sorted(history, key=lambda tpl: tpl[1])
 best = history[0]
print(f"best_params: {best[0]}, mae: {best[1]:.10f}" )<init_hyperparams>
all_df['Fence'] = all_df['Fence'].fillna('None' )
lgb_params = {
"task": "train",
"boosting_type": "gbdt",
"objective": "regression",
"eval_metric": "mae",
"num_leaves": best[0]["num_leaves"],
"learning_rate": 0.05,
"bagging_fraction": best[0]["bagging_fraction"],
"feature_fraction": best[0]["feature_fraction"],
"min_data_in_leaf": best[0]["min_data_in_leaf"],
"random_state": 93
}<train_model>
all_df['FireplaceQu'] = all_df['FireplaceQu'].fillna('None' )
pred_lgb1 = pd.DataFrame()
for k,(tr_id, vl_id)in enumerate(kf1.split(X, y)) :
X_train, X_val = X[tr_id, :], X[vl_id, :]
y_train, y_val = y[tr_id], y[vl_id]
lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_val, label=y_val, reference=lgb_train)
 model = lgb.train(params=lgb_params,
train_set=lgb_train,
valid_sets=(lgb_train, lgb_val),
num_boost_round=20000,
early_stopping_rounds=100,
verbose_eval=0)
 pred = model.predict(X_val, num_iteration=model.best_iteration)
 print(f"k={k+1}, mae: {mae(pred, y_val)}")
 pred = model.predict(X_test, num_iteration=model.best_iteration)
 pred = pd.Series(pred)
 pred_lgb1 = pd.concat([pred_lgb1, pred], axis=1 )<train_model>
all_df['LotFrontage'] = all_df.groupby('Neighborhood')['LotFrontage'].transform(
 lambda x: x.fillna(x.median()))
pred_lgb2 = pd.DataFrame()
for k,(tr_id, vl_id)in enumerate(kf2.split(X, y)) :
X_train, X_val = X[tr_id, :], X[vl_id, :]
y_train, y_val = y[tr_id], y[vl_id]
lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_val, label=y_val, reference=lgb_train)
 model = lgb.train(params=lgb_params,
train_set=lgb_train,
valid_sets=(lgb_train, lgb_val),
num_boost_round=20000,
early_stopping_rounds=100,
verbose_eval=0)
 pred = model.predict(X_val, num_iteration=model.best_iteration)
 print(f"k={k+1}, mae: {mae(pred, y_val)}")
 pred = model.predict(X_test, num_iteration=model.best_iteration)
 pred = pd.Series(pred)
 pred_lgb2 = pd.concat([pred_lgb2, pred], axis=1 )<concatenate>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_df[col] = all_df[col].fillna('None' )
pred = pd.concat([pred_lgb1, pred_lgb2], axis=1)
 pred = pred.mean(axis=1)
 pred<load_from_csv>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_df[col] = all_df[col].fillna(0 )
sample_sub = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv")
 sub = sample_sub.copy()
sub["time_to_eruption"] = pred
sub<save_to_csv>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_df[col] = all_df[col].fillna(0 )
sub.to_csv("submission.csv", index=False )<import_modules>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_df[col] = all_df[col].fillna('None' )
import numpy as np
import pandas as pd
from scipy.stats import skew, kurtosis
from scipy.signal import stft
import os
from tqdm.notebook import tqdm
import scipy
import re
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error as mse
import matplotlib.pyplot as plt
from xgboost import plot_importance
import xgboost as xgb 
<load_from_csv>
all_df['MasVnrType'] = all_df['MasVnrType'].fillna('None')
 all_df['MasVnrArea'] = all_df['MasVnrArea'].fillna(0 )
train = pd.read_csv('.. /input/predict-volcanic-eruptions-ingv-oe/train.csv')
<define_variables>
all_df['MSZoning'] = all_df['MSZoning'].fillna(all_df['MSZoning'].mode() [0] )
fs = 100
N = 60001
n = 256
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
DIR = '.. /input/predict-volcanic-eruptions-ingv-oe'<load_from_csv>
all_df['Utilities'] = all_df['Utilities'].fillna(all_df['Utilities'].mode() [0] )
def make_features(tgt):
tgt_df = train if tgt == 'train' else test
feature_set = []
for segment_id in tqdm(tgt_df['segment_id']):
segment_df = pd.read_csv(os.path.join(DIR,f'{tgt}/{segment_id}.csv'))
segment = [segment_id]
for sensor in segment_df.columns:
x = segment_df[sensor][:N]
if x.isna().sum() > 1000:
segment +=([np.NaN] * 10)
 continue
f, t, Z = scipy.signal.stft(x.fillna(0), fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
segment += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
feature_set.append(segment)
 cols = ['segment_id']
for i in range(10):
for j in ['A_pow', 'A_num','BH_pow', 'BH_num','BL_pow', 'BL_num','C_pow', 'C_num','D_pow', 'D_num']:
cols += [f's{i+1}_{j}']
feature_df = pd.DataFrame(feature_set, columns = cols)
 feature_df['segment_id'] = feature_df['segment_id'].astype('int')
 return feature_df<feature_engineering>
all_df['Functional'] = all_df['Functional'].fillna('Typ' )
feature_df = make_features('train' )<merge>
all_df['Electrical'] = all_df['Electrical'].fillna(all_df['Electrical'].mode() [0] )
train_set = pd.merge(train, feature_df, on = 'segment_id' )<feature_engineering>
all_df['KitchenQual'] = all_df['KitchenQual'].fillna(all_df['KitchenQual'].mode() [0] )
def create_columns(df,result):
df = df.fillna(0)
 for column in df:
result.at[index,'sum_'+column] = sum(df[column])
 result.at[index,'med_'+column] = np.median(df[column])
 result.at[index,'permiss_'+column] = df[column].isnull().sum() / df[column].size
result.at[index, 'skew_'+ column] = skew(df[column])
 result.at[index, 'kurtosis'+ column] = kurtosis(df[column])
 result.at[index,'max_'+column] = df[column].max()
result.at[index,'min_'+column] = df[column].min()
result.at[index, 'std_'+ column] = np.std(df[column])
 result.at[index, 'var_'+ column] = np.var(df[column])
 result.at[index,'quan_0.05'+ column] = np.quantile(df[column],0.05)
 result.at[index,'quan_0.1'+ column] = np.quantile(df[column],0.1)
 result.at[index,'quan_0.15'+ column] = np.quantile(df[column],0.15)
 result.at[index,'quan_0.2'+ column] = np.quantile(df[column],0.2)
 result.at[index,'quan_0.025'+ column] = np.quantile(df[column],0.25)
 result.at[index,'quan_0.3'+ column] = np.quantile(df[column],0.3)
 result.at[index,'quan_0.35'+ column] = np.quantile(df[column],0.35)
 result.at[index,'quan_0.4'+ column] = np.quantile(df[column],0.4)
 result.at[index,'quan_0.45'+ column] = np.quantile(df[column],0.45)
 result.at[index,'quan_0.5'+ column] = np.quantile(df[column],0.5)
 result.at[index,'quan_0.55'+ column] = np.quantile(df[column],0.55)
 result.at[index,'quan_0.6'+ column] = np.quantile(df[column],0.6)
 result.at[index,'quan_0.65'+ column] = np.quantile(df[column],0.65)
 result.at[index,'quan_0.7'+ column] = np.quantile(df[column],0.7)
 result.at[index,'quan_0.75'+ column] = np.quantile(df[column],0.75)
 result.at[index,'quan_0.8'+ column] = np.quantile(df[column],0.8)
 result.at[index,'quan_0.85'+ column] = np.quantile(df[column],0.85)
 result.at[index,'quan_0.9'+ column] = np.quantile(df[column],0.9)
 result.at[index,'quan_0.95'+ column] = np.quantile(df[column],0.95)
 return result 
<load_from_csv>
all_df['Exterior1st'] = all_df['Exterior1st'].fillna(all_df['Exterior1st'].mode() [0])
 all_df['Exterior2nd'] = all_df['Exterior2nd'].fillna(all_df['Exterior2nd'].mode() [0] )
for row in train_set.itertuples() :
index = row[0]
segmentid = row[1]
if str(segmentid)+".csv" in filenames_train:
df_segement = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/'+str(segmentid)+".csv")
 result = pd.concat([df_segement, df_segement.abs().add_suffix("_abs")], axis=1, join="inner")
 df_segemnt = result
train_set = create_columns(df_segement,train_set)
 if index % 100 == 0:
print(index)
<create_dataframe>
all_df['SaleType'] = all_df['SaleType'].fillna(all_df['SaleType'].mode() [0] )
test = pd.DataFrame([(re.findall("[0-9]+",a)) for a in filenames_test] , columns=['segment_id'] )<feature_engineering>
all_df['MSSubClass'] = all_df['MSSubClass'].fillna('None' )
feature_df_test = make_features('test')
<drop_column>

(all_df.isnull().sum() == 0 ).all()
feature_df_test.drop('segment_id', axis = 1, inplace = True )<merge>

 for col in all_df.select_dtypes(include=['float64'] ).columns:
all_df[col] = all_df[col].astype('int64' )
test_set = test.join(feature_df_test )<load_from_csv>

 for col in all_df.select_dtypes(include=['object'] ).columns:
all_df[col] = all_df[col].astype('category' )
for row in test_set.itertuples() :
index = row[0]
segementId = row[1]
if str(segementId)+".csv" in filenames_test:
df_segement = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/test/'+str(segementId)+".csv")
 result = pd.concat([df_segement, df_segement.abs().add_suffix("_abs")], axis=1, join="inner")
 df_segemnt = result
test_set = create_columns(df_segement,test_set)
 if index % 100 == 0:
print(index)
<save_to_csv>

 all_df['MSSubClass'] = all_df['MSSubClass'].astype('category')
 all_df['YrSold'] = all_df['YrSold'].astype('category')
 all_df['MoSold'] = all_df['MoSold'].astype('category' )
test_set.to_csv('test_work.csv' )<save_to_csv>

 train_df = pd.merge(all_df, train_df[['Id', 'SalePrice']], how='inner', on='Id')
 test_df = all_df[all_df.Id.isin(test_Id)]
train_set.to_csv('train_work.csv' )<prepare_x_and_y>

 d = defaultdict(int)
 df = pd.DataFrame()
for col, dtype in zip(all_df.columns, all_df.dtypes.values):
description = ''
if type(dtype)== CategoricalDtype:
description = dtype.categories
dtype = 'category'
df = df.append({'column': col, 'dtype': dtype, 'description': description}, ignore_index=True)
 d[dtype] += 1
print(d)
 df.set_index('column')[['dtype', 'description']]
y_train = train_set['time_to_eruption']
train_df = train_set.drop(['time_to_eruption','segment_id'], axis = 1 )<compute_test_metric>

 num_uniq = all_df.select_dtypes(include=['category'] ).nunique().sort_values()
index = num_uniq.index
num_uniq
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})" 
.format(results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("" )<train_on_grid>

 num_uniq = all_df.select_dtypes(include=['int64'] ).nunique().sort_values()
index = num_uniq.index
num_uniq
param = {'eta': [0.05,0.1,0.2,0.3],
'max_depth': [4,5,6,7,8,9,10],
'subsample ': [0.5,0.75,1],
'gamma': [0.05,0.075,0.09,0.1,0.15,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
}
search = RandomizedSearchCV(xgb.XGBRegressor() ,param,n_iter = 30,cv = 5, scoring = 'neg_mean_squared_error')
 search.fit(train_df,y_train)
<init_hyperparams>

 model_lgb = lgb.LGBMRegressor(num_leaves=10,
max_depth=5,
learning_rate=0.1,
random_state=2)
 model_lgb.fit(train_df.drop(['Id', 'SalePrice'], axis=1), train_df.SalePrice )
params = {'subsample': 0.75, 'max_depth': 9, 'gamma': 0.1, 'eta': 0.1}<train_model>
rmsle_cv(model_lgb, train_df.drop('SalePrice', axis=1), train_df.SalePrice )
xgb_reg = xgb.XGBRegressor(**params)
 xgb_reg.fit(train_df, y_train )<features_selection>
df = pd.DataFrame(model_lgb.feature_importances_,
index=train_df.drop(['Id', 'SalePrice'], axis=1 ).columns,
columns=['importance'] ).sort_values('importance', ascending=False)
 df[df.importance > 10]
sorted_idx = xgb_reg.feature_importances_.argsort()
imp = xgb_reg.feature_importances_
col = train_df.columns<define_variables>

 imp_index = pd.Index([
'GrLivArea',
'TotalBsmtSF',
'LotArea',
'Neighborhood',
'OverallQual',
'OverallCond',
'GarageCars',
'MasVnrArea',
'YearBuilt',
'KitchenQual',
'YearRemodAdd',
'SalePrice',
] )
todrop = col[sorted_idx[:50]]<drop_column>

 train_df.loc[:, imp_index].nunique()
train_df.drop(todrop, axis = 1, inplace = True)
 test_set.drop(todrop, axis = 1, inplace = True )<train_model>
c_train_df = train_df.copy()
c_test_df = test_df.copy()
c_all_df = all_df.copy()
xgb_reg = xgb.XGBRegressor(**params)
 xgb_reg.fit(train_df, y_train )<drop_column>

 imp_index = pd.Index([
'GrLivArea',
'TotalBsmtSF',
'LotArea',
'Neighborhood',
'OverallQual',
'OverallCond',
'GarageCars',
'MasVnrArea',
'YearBuilt',
'KitchenQual',
'YearRemodAdd',
'SalePrice',
] )
test_df = test_set.drop('segment_id', axis = 1 )<create_dataframe>

 doubtful_train_df = train_df[ 
(train_df.SalePrice > 700000)\
|(( train_df.GrLivArea > 4000)&(train_df.SalePrice < 200000))
|(( train_df.LotArea > 100000))
]
doubtful_train_df.loc[:, imp_index].sort_values('SalePrice', ascending=False )
submission = pd.DataFrame(test_set['segment_id'], columns=['segment_id'] )<predict_on_test>

 c_all_df = c_all_df.drop(
 c_all_df[c_all_df.Id.isin(
 train_df[ 
(train_df.SalePrice > 700000)\
|(( train_df.GrLivArea > 4000)&(train_df.SalePrice < 200000))
].Id)].index
)
submission['time_to_eruption'] = xgb_reg.predict(test_df )<prepare_output>
c_all_df['YearsSinceBuilt'] = c_all_df.YrSold.astype('int64')- c_all_df.YearBuilt
c_all_df.loc[c_all_df.YearsSinceBuilt <= 0, 'YearsSinceBuilt'] = 0
submission.set_index('segment_id', inplace = True )<save_to_csv>
c_all_df['YearsSinceRemod'] = c_all_df.YrSold.astype('int64')- c_all_df.YearRemodAdd
c_all_df.loc[c_all_df.YearRemodAdd == 1950, 'YearsSinceRemod'] = \
c_all_df.loc[c_all_df.YearRemodAdd == 1950, 'YearsSinceBuilt']
c_all_df.loc[c_all_df.YearsSinceRemod <= 0, 'YearsSinceRemod'] = 0
submission.to_csv('out.csv' )<load_from_csv>
c_all_df['TotalSF'] = c_all_df['TotalBsmtSF'] + c_all_df['1stFlrSF'] + c_all_df['2ndFlrSF']
X = pd.read_csv(".. /input/preprocessed-volcano-data/X.csv",sep=";",index_col=0)
 Y = pd.read_csv(".. /input/preprocessed-volcano-data/Y.csv",sep=";",index_col=0 )<train_model>
c_all_df['TotalBathrooms'] = \
c_all_df['FullBath'] +(0.5 * c_all_df['HalfBath'])+ \
c_all_df['BsmtFullBath'] +(0.5 * c_all_df['BsmtHalfBath'] )
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X,Y["time_to_eruption"],test_size=500,random_state=1234)
 reg = LinearRegression()
cols = [col for col in X.columns if "max" in col and not "argmax" in col]
reg.fit(Xtrain[cols].values, Ytrain)
 pred = reg.predict(Xtest[cols].values)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(Ytest,pred)[0,1]**2
print(R2 )<train_on_grid>
c_all_df['TotalPorchSf'] = \
c_all_df['OpenPorchSF'] + c_all_df['3SsnPorch'] + \
c_all_df['EnclosedPorch'] + c_all_df['ScreenPorch'] + c_all_df['WoodDeckSF']
gridsearch = GridSearchCV(estimator=RandomForestRegressor() ,
param_grid={"max_features" : [50, 100, 150],
"min_samples_leaf" : [1, 5],
"n_estimators" : [1000]},
cv=5,n_jobs=5,scoring="neg_mean_absolute_error",verbose=10)
 gridsearch.fit(Xtrain, Ytrain)
 res_RF = pd.DataFrame({"max_features" : [list(x.values())[0] for x in gridsearch.cv_results_["params"]],
"min_samples_leaf" : [list(x.values())[1] for x in gridsearch.cv_results_["params"]],
"mean_mae" : np.round(gridsearch.cv_results_["mean_test_score"],4),
"sd_mae" : np.round(gridsearch.cv_results_["std_test_score"],4)})
 print(res_RF )<train_on_grid>
gridsearch = GridSearchCV(estimator=ExtraTreesRegressor() ,
param_grid={"max_features" : [50, 100, 150],
"min_samples_leaf" : [1, 5],
"n_estimators" : [1000]},
cv=5,n_jobs=5,scoring="neg_mean_absolute_error",verbose=10)
 gridsearch.fit(Xtrain, Ytrain)
 res_ET = pd.DataFrame({"max_features" : [list(x.values())[0] for x in gridsearch.cv_results_["params"]],
"min_samples_leaf" : [list(x.values())[1] for x in gridsearch.cv_results_["params"]],
"mean_mae" : np.round(gridsearch.cv_results_["mean_test_score"],4),
"sd_mae" : np.round(gridsearch.cv_results_["std_test_score"],4)})
 print(res_ET )<compute_test_metric>
ET = ExtraTreesRegressor(max_features=164,min_samples_leaf=1)
 ET.fit(Xtrain, Ytrain)
 pred = ET.predict(Xtest)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(Ytest,pred)[0,1]**2
print(R2 )<features_selection>
c_all_df = c_all_df.drop(columns=['Street', 'PoolQC', 'Utilities'],axis=1 )
def FeatureSelection(X,Y,nfolds=5,random_state=123):
XX = X.values
n_iter = 30
cvmeanMAE = np.zeros(n_iter)
 cvminMAE = np.zeros(n_iter)
 cvmaxMAE = np.zeros(n_iter)
 cols = X.columns
remaining_cols = []
cvsplit = KFold(n_splits=nfolds, shuffle=True, random_state=random_state)
 for i in range(n_iter):
print("Variable selection step " + str(i + 1)+ " - " + str(XX.shape[1])+ " features remaining")
 rf = ExtraTreesRegressor(max_features=XX.shape[1], min_samples_leaf=1, n_estimators=1000)
 cv = cross_validate(estimator=rf, X=XX, y=Y, cv=cvsplit, n_jobs=5,
scoring="neg_mean_absolute_error")
 cvmeanMAE[i] = np.mean(-cv["test_score"])
 cvminMAE[i] = np.min(-cv["test_score"])
 cvmaxMAE[i] = np.max(-cv["test_score"])
 rf.fit(XX, Y)
 varimp = rf.feature_importances_
ind = np.argsort(varimp)[5:]
XX = XX[:, ind]
cols = cols[ind]
remaining_cols.append(cols)
 res = {"remaining_cols" : remaining_cols,
"cvmeanMAE" : cvmeanMAE,
"cvminMAE" : cvminMAE,
"cvmaxMAE" : cvmaxMAE}
return res
tmp = FeatureSelection(Xtrain, Ytrain)
 pyplot.bar(np.arange(len(tmp["cvmeanMAE"])) ,height=tmp["cvmeanMAE"],
yerr=np.concatenate([(tmp["cvmeanMAE"]-tmp["cvminMAE"] ).reshape(-1,1), 
(tmp["cvmaxMAE"]-tmp["cvmeanMAE"] ).reshape(-1,1)],axis=1 ).T )<define_variables>

 c_all_df.LotShape.replace(to_replace=['IR3', 'IR2', 'IR1', 'Reg'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.LandContour.replace(to_replace=['Low', 'Bnk', 'HLS', 'Lvl'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.LandSlope.replace(to_replace=['Sev', 'Mod', 'Gtl'], value=[0, 1, 2], inplace=True)
 c_all_df.ExterQual.replace(to_replace=['Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.ExterCond.replace(to_replace=['Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtQual.replace(to_replace=['None', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtCond.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtExposure.replace(to_replace=['None', 'No', 'Mn', 'Av', 'Gd'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtFinType1.replace(to_replace=['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], value=[0, 1, 2, 3, 4, 5, 6], inplace=True)
 c_all_df.BsmtFinType2.replace(to_replace=['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], value=[0, 1, 2, 3, 4, 5, 6], inplace=True)
 c_all_df.HeatingQC.replace(to_replace=['Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.Electrical.replace(to_replace=['Mix', 'FuseP', 'FuseF', 'FuseA', 'SBrkr'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.KitchenQual.replace(to_replace=['Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.Functional.replace(to_replace=['Sev', 'Maj2', 'Maj1', 'Mod', 'Min2', 'Min1', 'Typ'], value=[0, 1, 2, 3, 4, 5, 6], inplace=True)
 c_all_df.FireplaceQu.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4, 5], inplace=True)
 c_all_df.GarageFinish.replace(to_replace=['None', 'Unf', 'RFn', 'Fin'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.GarageQual.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4, 5], inplace=True)
 c_all_df.GarageCond.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4, 5], inplace=True)
 c_all_df.PavedDrive.replace(to_replace=['N', 'P', 'Y'], value=[0, 1, 2], inplace=True)
 c_all_df.Fence.replace(to_replace=['None', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'], value=[0, 1, 2, 3, 4], inplace=True )
sel_cols = ['S1_max', 'S10_Q75', 'S3_Q50', 'S1_sumNA', 'S3_Q1', 'S3_max', 'S9_std',
'S6_Q5', 'S7_Q1', 'S10_Q95', 'S5_Q90', 'S2_Q5', 'S5_Q5', 'S6_Q10',
'S7_Q50', 'S1_Q95', 'S3_Q75', 'S5_Q10', 'S8_Q99', 'S4_Q95', 'S3_sumNA',
'S7_Q90', 'S8_Q50', 'S6_Q90', 'S10_Q90', 'S2_sumNA', 'S4_Q99', 'S2_Q75',
'S9_Q90', 'S9_Q25', 'S1_Q99', 'S6_Q1', 'S1_Q5', 'S7_Q99', 'S3_Q5',
'S3_Q95', 'S5_Q99', 'S4_Q1', 'S9_Q50', 'S2_std', 'S1_Q75', 'S2_Q50',
'S7_Q5', 'S10_Q25', 'S5_sumNA', 'S8_Q5', 'S10_Q1']<train_model>
c_all_df.MSSubClass = c_all_df.MSSubClass.astype('int64')
 c_all_df.loc[c_all_df['MSSubClass']==60, 'MSSubClass'] = 0
c_all_df.loc[(c_all_df['MSSubClass']==20)|(c_all_df['MSSubClass']==120), 'MSSubClass'] = 1
c_all_df.loc[c_all_df['MSSubClass']==75, 'MSSubClass'] = 2
c_all_df.loc[(c_all_df['MSSubClass']==40)|(c_all_df['MSSubClass']==70)|(c_all_df['MSSubClass']==80), 'MSSubClass'] = 3
c_all_df.loc[(c_all_df['MSSubClass']==50)|(c_all_df['MSSubClass']==85)|(c_all_df['MSSubClass']==90)|(c_all_df['MSSubClass']==160)|(c_all_df['MSSubClass']==190), 'MSSubClass'] = 4
c_all_df.loc[(c_all_df['MSSubClass']==30)|(c_all_df['MSSubClass']==45)|(c_all_df['MSSubClass']==180), 'MSSubClass'] = 5
c_all_df.loc[(c_all_df['MSSubClass']==150), 'MSSubClass'] = 6
Xtrain = Xtrain[sel_cols]
Xtest = Xtest[sel_cols]
ET = ExtraTreesRegressor(max_features=47,min_samples_leaf=1,n_estimators=1000)
 ET.fit(Xtrain, Ytrain)
 pred = ET.predict(Xtest)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(pred,Ytest)[0,1]**2
print(R2 )<choose_model_class>
c_all_df.GarageCars.replace(to_replace=[0, 1, 2, 3, 4, 5], value=[0, 1, 2, 3, 3, 3], inplace=True )
class Network:
def __init__(self,file,nval=500,nepochs=300,batch_size=64):
self.nval = nval
self.nepochs = nepochs
self.batch_size = batch_size
self.file = file
def __build_model(self,input_shape):
net = keras.models.Sequential([
keras.layers.Input(shape=input_shape),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(512, activation="relu"),
keras.layers.Dropout(0.5),
keras.layers.Dense(1, activation="linear")
 ])
 return net
def fit(self,X,Y):
Xtrain, Xval, Ytrain, Yval = train_test_split(X, Y, test_size=self.nval)
 scaler = StandardScaler()
scaler.fit(Xtrain)
 Xtrain_sc = scaler.transform(Xtrain)
 Xval_sc = scaler.transform(Xval)
 my = np.mean(Ytrain)
 sy = np.std(Ytrain)
 Ytrain_sc =(Ytrain - my)/ sy
Yval_sc =(Yval - my)/ sy
net = self.__build_model(X.shape[1])
 checkpoint = keras.callbacks.ModelCheckpoint(self.file,monitor="val_loss",
save_best_only=True,mode="min",
verbose=1)
 lr_callback = keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=20)
 net.compile(loss="mean_absolute_error", optimizer=keras.optimizers.Nadam(learning_rate=0.001))
fitted = net.fit(x=Xtrain_sc, y=Ytrain_sc, batch_size=self.batch_size,
epochs=self.nepochs, verbose=1,
callbacks=[lr_callback, checkpoint],
validation_data=(Xval_sc, Yval_sc))
net = keras.models.load_model(self.file)
 fitted = {"net" : net, "scaler" : scaler, "my" : my, "sy" : sy}
self.fitted = fitted
def predict(self,Xtest):
fitted = self.fitted
net = fitted["net"]
scaler = fitted["scaler"]
my = fitted["my"]
sy = fitted["sy"]
pred = net.predict(scaler.transform(Xtest)) [:,0]*sy+my
return pred<compute_test_metric>
skewness = c_all_df.select_dtypes(include=['int64'] ).apply(lambda x: skew(x))
skew_index = skewness[abs(skewness)>= 0.75].index
skewness[skew_index].sort_values(ascending=False )
net = Network(file="fitted_network.h5",nval=500,nepochs=300,batch_size=64)
 net.fit(Xtrain,Ytrain)
 pred = net.predict(Xtest)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(Ytest,pred)[0,1]**2
print(R2 )<load_from_csv>

 lam = 0.15
for column in skew_index:
c_all_df[column] = boxcox1p(c_all_df[column], lam )
X = pd.read_csv(".. /input/preprocessed-volcano-data/X.csv",sep=";",index_col=0)
 Y = pd.read_csv(".. /input/preprocessed-volcano-data/Y.csv",sep=";",index_col=0)
 Xtest = pd.read_csv(".. /input/preprocessed-volcano-data/Xtest.csv",sep=";",index_col=0)
 segment_id = Xtest.index
X = X[sel_cols]
Xtest = Xtest[sel_cols]
net = Network(file="fitted_network.h5",nval=500,nepochs=300,batch_size=64)
 net.fit(X,Y["time_to_eruption"])
 pred = net.predict(Xtest)
 res = pd.DataFrame({"segment_id" : segment_id, "time_to_eruption" : pred})
 res.to_csv("prediction_test.csv",sep=",",columns=res.columns,index=False )<import_modules>

(all_df.isnull().sum() == 0 ).all()
import tensorflow as tf
from tensorflow.keras import Model, Input, models
from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, ReduceLROnPlateau
from tensorflow.keras.layers import BatchNormalization, Dropout, Flatten, Dense, InputLayer, Concatenate, Add, SeparableConv2D, Layer
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.initializers import TruncatedNormal
from sklearn.model_selection import RepeatedKFold
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd<load_from_csv>

 c_train_df = pd.merge(c_all_df, train_df[['Id', 'SalePrice']], how='inner', on='Id')
 c_test_df = c_all_df[c_all_df.Id.isin(test_Id)]
train = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')
 sample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')
 train_ids, test_ids = train['segment_id'], sample_submission['segment_id']
y = train['time_to_eruption']
X = np.load('/kaggle/input/volcano-pca/train.npy')
 X_test = np.load('/kaggle/input/volcano-pca/test.npy')
 X_rolling = np.load('/kaggle/input/volcano-pca/rolling_train.npy')
 X_rolling_test = np.load('/kaggle/input/volcano-pca/rolling_test.npy')
 X_concat = np.load('/kaggle/input/volcano-pca/train_concat.npy')
 X_aug = np.load('/kaggle/input/volcano-pca/aug_train_concat_2.npy')
 y_aug = np.load('/kaggle/input/volcano-pca/y_aug2.npy')
 X = X_aug[:, :, :16]
X_rolling = X_aug[:, :, 16:]
MODEL_NAME = 'res32_v3'
In1 = Input(shape=(10, 16, 1))
In2 = Input(shape=(10, 16, 1))
Inm = Input(shape=(10, 32, 1))<define_search_model>
WInit = TruncatedNormal(0, 1, 111)
 def concat_inputs(Inp1, Inp2):

 return Concatenate(axis=2 )([Inp1, Inp2])
 def fc(Inp, unit=100, n=3, activation='relu', pred=True, bn=False):

 Inp = Flatten()(Inp)
 out = Dense(unit, activation=activation ,kernel_initializer=WInit )(Inp)
 for i in range(0, n-2):
out = Dense(unit, activation=activation ,kernel_initializer=WInit )(out)
 if bn: out = BatchNormalization()(out)
 if not pred:
out = Dense(unit, activation=activation ,kernel_initializer=WInit )(out)
 return out
out = Dense(1, kernel_initializer=TruncatedNormal(10, 1e-1, 11))(out)
 return out
def submit_prediction(pred, write=True, file_name="", Return=True):

 pred = np.where(pred < 0, np.mean(pred), pred)
 sample_submission['time_to_eruption'] = pred
if write: sample_submission.to_csv(f'{file_name}.csv', index=False)
 if Return: return pred
def vertical_kernel(x,y, length): return [(y, x + i)for i in range(length)]
def horizontal_kernel(x,y, length): return [(y + i, x)for i in range(length)]
def module(inp, vertical_kernels, horizontal_kernels, unit=16, std=1e-1, seed=11):
vertical_convs = [SeparableConv2D(unit,
kernel_size,
kernel_initializer=TruncatedNormal(0, std, seed),
bias_initializer=TruncatedNormal(0, std, seed))(inp)for kernel_size in vertical_kernels]
horizontal_convs = [SeparableConv2D(unit,
kernel_size,
kernel_initializer=TruncatedNormal(0, std, seed),
bias_initializer=TruncatedNormal(0, std, seed))(inp)for kernel_size in horizontal_kernels]
vertical_concats = [Concatenate(axis=2 )(
 [vertical_convs[i], vertical_convs[len(vertical_convs)- 1 - i]] 
)for i in range(len(vertical_convs)// 2)]
horizontal_concats = [Concatenate(axis=1 )(
 [horizontal_convs[i], horizontal_convs[len(horizontal_convs)- 1 - i]] 
)for i in range(len(horizontal_convs)// 2)]
return vertical_concats, horizontal_concats<categorify>

 c_train_X = c_train_df.drop(['Id', 'SalePrice'], axis=1)
 c_train_y = c_train_df.SalePrice
c_test_X = c_test_df.drop('Id', axis=1)

d_c_train_X = pd.get_dummies(c_train_X)
 d_c_test_X = pd.get_dummies(c_test_X )
def model2(inp):
vertical_kernels = vertical_kernel(6,10, 8)
 horizontal_kernels = horizontal_kernel(32, 3, 8)
 vert_concat1, hor_concat1 = module(inp, vertical_kernels, horizontal_kernels)
 vert_preds = [fc(layer, unit=100, n=5, activation='relu', pred=True)for layer in vert_concat1]
hor_preds = [fc(layer, unit=100, n=5, activation='relu', pred=True)for layer in hor_concat1]
vert_preds.extend(hor_preds)
 pred = fc(Concatenate()(vert_preds), unit=25, n=3, activation='relu', pred=True)
 return pred<define_search_model>

 def tune_hyperparameters(model, param_grid, train_X, train_y):
grid = GridSearchCV(
 model, param_grid,
scoring='neg_mean_squared_log_error',
cv=5, n_jobs=-1, 
)
 grid.fit(train_X, train_y)
 best_params = grid.best_params_
best_score = np.round(np.sqrt(-1 * grid.best_score_), 5)
 return best_params, best_score
def resnet_module(inp, unit=32, stride1=(3,3), stride2=(5,5), stride3=(3,3), mid_activation=None, padding='same', bn=True, std=1e-2, seed=123):
l1 = SeparableConv2D(filters=unit,
kernel_size=stride1,
kernel_initializer=TruncatedNormal(0, std, seed),
padding=padding
)(inp)
 l2 = SeparableConv2D(filters=unit,
kernel_size=stride2,
kernel_initializer=TruncatedNormal(0, std*1.1, seed),
activation=mid_activation,
padding='same'
)(l1)
 l3 = SeparableConv2D(filters=unit,
kernel_size=stride3,
kernel_initializer=TruncatedNormal(0, std*5, seed),
padding=padding
)(l2)
 la = Add()([l1, l3])
 if bn: return BatchNormalization()(la)
 return la
def mres(inp):
b1 = resnet_module(inp, stride1=(1,3), stride2=(5,5), stride3=(3,1), bn=False, std=1)
 b1 = resnet_module(b1, stride1=(3,1), stride2=(3,3), stride3=(1,3), mid_activation='relu',bn=True, std=1.5)
 b1 = resnet_module(b1, stride1=(3,3), stride2=(3,3), stride3=(3,3), mid_activation='relu',bn=True, std=1)
 b2 = resnet_module(b1, unit=64, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='relu', bn=False, std=1e-2)
 b2 = resnet_module(b2, unit=64, stride1=(6,1), stride2=(3,3), stride3=(1,6),bn=False, std=1.5e-3)
 b2 = resnet_module(b2, unit=64, stride1=(4,4), stride2=(3,3), stride3=(4,4), mid_activation='relu',bn=False, std=3e-1)
 b3 = resnet_module(b2, unit=128, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid', bn=False, std=1e-2)
 b3 = resnet_module(b3, unit=128, stride1=(6,1), stride2=(3,3), stride3=(1,6),bn=False, std=1.5e-3)
 b3 = resnet_module(b3, unit=128, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid',bn=True, std=3e-1)
 b3 = Dropout(0.5 )(b3)
 b4 = resnet_module(b3, unit=256, stride1=(3,3), stride2=(1,1), stride3=(3,3), mid_activation='sigmoid', bn=False, std=1e-2)
 b4 = resnet_module(b4, unit=256, stride1=(6,1), stride2=(1,1), stride3=(1,6),bn=False, std=1.5e-3)
 b4 = resnet_module(b4, unit=256, stride1=(2,3), stride2=(1,1), stride3=(2,1), mid_activation='relu',bn=True, std=3e-1)
 b5 = resnet_module(b4, unit=256, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid', bn=True, std=1e-2)
 b5 = resnet_module(b5, unit=256, stride1=(3, 3), stride2=(3,3), stride3=(3,3),bn=False, std=1.5e-3)
 b5 = resnet_module(b5, unit=256, stride1=(4,4), stride2=(3,3), stride3=(4,4), mid_activation='relu',bn=True, std=3e-1)
 b5 = Dropout(0.5 )(b5)
 b6 = resnet_module(b5, unit=256, stride1=(3,3), stride2=(7,7), stride3=(2,3), mid_activation='relu', bn=True, std=1e-2)
 b6 = resnet_module(b6, unit=256, stride1=(4,1), stride2=(3,3), stride3=(1,4),bn=False, std=1.5e-3)
 b6 = resnet_module(b6, unit=256, stride1=(2,2), stride2=(1,1), stride3=(3,2), mid_activation='relu',bn=True, std=3e-1)
 p = fc(b6, unit=100, n=3, activation='relu')
 return p<set_options>
model = make_pipeline(
 RobustScaler() ,
LinearRegression() , 
)
 lr_model = MyEstimator(model)
 lr_model.fit(d_c_train_X, c_train_y)
 lr_eval = evaluation_model(lr_model, d_c_train_X, c_train_y, d_c_test_X)
 print(lr_eval )
model = models.load_model('/kaggle/input/volcanonn/res32_v2')
 K.set_value(model.optimizer.learning_rate, 0.0015 )<find_best_params>
model = make_pipeline(
 RobustScaler() ,
LassoCV(
 alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10), 
), 
)
 lasso_cv_model = MyEstimator(model)
 lasso_cv_model.fit(d_c_train_X, c_train_y)
 lasso_cv_eval = evaluation_model(lasso_cv_model, d_c_train_X, c_train_y, d_c_test_X)
 print(lasso_cv_eval)
 opt_alpha = lasso_cv_model.model.steps[1][1].alpha_
print(f'
opt_alpha: {opt_alpha}' )
model.optimizer.learning_rate<train_model>
model = make_pipeline(
 RobustScaler() ,
RidgeCV(
 alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10), 
), 
)
 ridge_cv_model = MyEstimator(model)
 ridge_cv_model.fit(d_c_train_X, c_train_y)
 ridge_cv_eval = evaluation_model(ridge_cv_model, d_c_train_X, c_train_y, d_c_test_X)
 print(ridge_cv_eval)
 opt_alpha = ridge_cv_model.model.steps[1][1].alpha_
print(f'
opt_alpha: {opt_alpha}' )
def kfold_validation(model, X, y, X_test, n_rep=3, n_fold=5, batch_size=32, epochs=300, verbose=1):

 history, index = {}, 0
prediction = np.zeros(( X_test.shape[0]))
preds = np.empty(( n_rep * n_fold, X_test.shape[0]))
kf = RepeatedKFold(n_splits=n_fold, n_repeats=n_rep, random_state=288490)
 for train_indices, val_indices in kf.split(X, y):
print(f'{index + 1}th fold, Validation Indices: ', val_indices[:5])
 X_train, X_val = X[train_indices], X[val_indices]
X_rolling_train, X_rolling_val = X_rolling[train_indices], X_rolling[val_indices]
y_train, y_val = y[train_indices], y[val_indices]
history[index] = model.fit(
 x=[X_train, X_rolling_train],
y=y_train,
epochs=epochs,
batch_size=batch_size,
validation_data=([X_val, X_rolling_val], y_val),
verbose=verbose,
callbacks=[
ReduceLROnPlateau(factor=0.96, patience=10, verbose=True, monitor='val_loss', min_lr=1e-45, min_delta=1e-4),
EarlyStopping(
 monitor='val_loss',
patience=25,
restore_best_weights=True,
min_delta=1e4,
verbose=True 
)
 ] 
)
 model_prediction = model.predict([X_test, X_rolling_test],
batch_size=batch_size,
verbose=False ).reshape(X_test.shape[0])
 model_prediction = submit_prediction(model_prediction,
write=True,
file_name=f"sub{index}",
Return=True)
 preds[index] = model_prediction
index += 1
print('
p = pd.DataFrame(preds)
 p = p.sum() /(n_fold * n_rep)
 avg_pred = submit_prediction(p,
write=True,
file_name=f"S_avg",
Return=True)
 return model, preds, avg_pred, history<compute_train_metric>
model = make_pipeline(
 RobustScaler() ,
ElasticNetCV(
 alphas=(0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007),
l1_ratio=(0.8, 0.85, 0.9, 0.95, 0.99, 1), 
), 
)
 elsnt_cv_model = MyEstimator(model)
 elsnt_cv_model.fit(d_c_train_X, c_train_y)
 elsnt_cv_eval = evaluation_model(elsnt_cv_model, d_c_train_X, c_train_y, d_c_test_X)
 print(elsnt_cv_eval)
 opt_alpha = elsnt_cv_model.model.steps[1][1].alpha_
opt_l1_ratio = elsnt_cv_model.model.steps[1][1].l1_ratio_
print(f'
opt_alpha: {opt_alpha} opt_l1_ratio: {opt_l1_ratio}' )
model, preds, avg_pred, history = kfold_validation(model, X, y_aug, X_test, 6, 3, 32, epochs=500, verbose=0)
 np.save('preds_V16.npy', preds)
 model.save(f'/kaggle/working/{MODEL_NAME}' )<import_modules>
model = make_pipeline(
 RobustScaler() ,
SVR() , 
)
 svr_model = MyEstimator(model)
 param_grid = {'model__svr__C': [1, 10, 20],
'model__svr__epsilon': [0.001, 0.01, 0.1],
'model__svr__gamma': [0.0001, 0.001, 0.01]}
best_params, best_score = \
tune_hyperparameters(svr_model, param_grid, d_c_train_X, c_train_y)
 svr_model.set_params(**best_params)
 svr_model.fit(d_c_train_X, c_train_y)
 svr_eval = evaluation_model(svr_model, d_c_train_X, c_train_y, d_c_test_X)
 print(svr_eval)
 print(f'
grid best_params: {best_params}' )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from glob import glob
import os
import dask.dataframe as dd
from tsfresh.feature_extraction import feature_calculators
import librosa
import pywt
from joblib import Parallel, delayed
from tqdm import tqdm_notebook
import scipy as sp
import itertools
import gc
from sklearn.metrics import mean_absolute_error
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from xgboost import XGBRegressor
import lightgbm as lgb
import gc 
<load_from_csv>
model = make_pipeline(
 RobustScaler() ,
KernelRidge() , 
)
 kr_model = MyEstimator(model)
 param_grid = {'model__kernelridge__alpha': [0.01, 0.1, 0.5, 1],
'model__kernelridge__kernel': ['linear', 'polynomial'],
'model__kernelridge__degree': [1, 1.5, 2, 3],
'model__kernelridge__coef0': [3, 4, 5]}
best_params, best_score = \
tune_hyperparameters(kr_model, param_grid, d_c_train_X, c_train_y)
 kr_model.set_params(**best_params)
 kr_model.fit(d_c_train_X, c_train_y)
 kr_eval = evaluation_model(kr_model, d_c_train_X, c_train_y, d_c_test_X)
 print(kr_eval)
 print(f'
grid best_params: {best_params}' )
train = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')
 train['segment_id'] = train['segment_id'].astype(str)
 train = train.sort_values('segment_id')
 y = train['time_to_eruption']
del(train )<define_variables>
model = make_pipeline(
 RobustScaler() ,
KNeighborsRegressor() , 
)
 knn_model = MyEstimator(model)
 param_grid = {'model__kneighborsregressor__n_neighbors': [3, 5, 7]}
best_params, best_score = \
tune_hyperparameters(knn_model, param_grid, d_c_train_X, c_train_y)
 knn_model.set_params(**best_params)
 knn_model.fit(d_c_train_X, c_train_y)
 knn_eval = evaluation_model(knn_model, d_c_train_X, c_train_y, d_c_test_X)
 print(knn_eval)
 print(f'
grid best_params: {best_params}' )
train_paths = sorted(glob('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/*'))<load_from_csv>
model = ExtraTreesRegressor(n_jobs=-1, n_estimators=100)
 et_model = MyEstimator(model)
 param_grid = {'model__max_depth': [1, 3, 5]}
best_params, best_score = \
tune_hyperparameters(et_model, param_grid, d_c_train_X, c_train_y)
 et_model.set_params(**best_params)
 et_model.fit(d_c_train_X, c_train_y)
 et_eval = evaluation_model(et_model, d_c_train_X, c_train_y, d_c_test_X)
 print(et_eval)
 print(f'
grid best_params: {best_params}' )
sub = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv' )<statistical_test>
model = XGBRegressor(learning_rate=0.01, n_estimators=3000,
max_depth=3,
nthread=-1,
objective='reg:squarederror',
min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1,
reg_alpha=0.00006)
 xgb_model = MyEstimator(model)
 xgb_model.fit(d_c_train_X, c_train_y)
 xgb_eval = evaluation_model(xgb_model, d_c_train_X, c_train_y, d_c_test_X)
 print(xgb_eval )
np.random.seed(1337)
 noise = np.random.normal(-10, 200, 60_001)
 def denoise_signal_simple(x, wavelet='db4', level=1):
coeff = pywt.wavedec(x, wavelet, mode="per")
 uthresh = 10
coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per')
 def feature_gen(path):
X =(pd.read_csv(path ).fillna(200 ).values.T + noise ).T
z = X - np.median(X,axis=0)
 features = []
for i in range(X.shape[1]):
sig = z[:,i]
den_sample_simple = denoise_signal_simple(sig)
 mfcc = librosa.feature.mfcc(sig)
 mfcc_mean = mfcc.mean(axis=1)
 percentile_roll50_std_20 = np.percentile(pd.Series(sig ).rolling(50 ).std().dropna().values, 20)
 features.extend([feature_calculators.number_peaks(den_sample_simple, 2),percentile_roll50_std_20,mfcc_mean[18],mfcc_mean[4]])
 return features<set_options>
model = LGBMRegressor(learning_rate=0.01, n_estimators=3000,
num_leaves=5,
objective='regression',
max_bin=55, bagging_fraction=0.8,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=6, min_sum_hessian_in_leaf=11)
 lgb_model = MyEstimator(model)
 lgb_model.fit(d_c_train_X, c_train_y)
 lgb_eval = evaluation_model(lgb_model, d_c_train_X, c_train_y, d_c_test_X)
 print(lgb_eval )
train_samples = np.array(train_samples ).reshape(-1,40)
 del(train_paths)
 gc.collect()<train_model>
models = [ 
('lr_model', lr_eval), 
('lasso_cv_model', lasso_cv_eval), 
('ridge_cv_model', ridge_cv_eval), 
('elsnt_cv_model', elsnt_cv_eval), 
('svr_model', svr_eval), 
('kr_model', kr_eval), 
('knn_model', knn_eval), 
('et_model', et_eval), 
('xgb_model', xgb_eval), 
('lgb_model', lgb_eval),
]
for mname, meval in models:
print(f'[{mname}]')
 print(meval )
cat = CatBoostRegressor(loss_function='MAE',iterations = 5000)
 cat.fit(train_samples,y )<compute_test_metric>
stack_cv_model = StackingCVRegressor(
 regressors=(ridge_cv_model, lasso_cv_model, elsnt_cv_model,
svr_model, kr_model, xgb_model, lgb_model),
meta_regressor=lgb_model,
cv=5,
n_jobs=-1,
use_features_in_secondary=True)
 stack_cv_model.fit(np.array(d_c_train_X), np.array(c_train_y))
stack_cv_eval = evaluation_model(stack_cv_model, np.array(d_c_train_X), np.array(c_train_y), np.array(d_c_test_X))
print(stack_cv_eval )
<define_variables><EOS> 
def output_submission_file(test_X, filename='submission.csv'):
prediction = blend_models_predict(test_X)
 df = pd.DataFrame({'Id': test_Id.values, 'SalePrice': prediction})
 print(f'{df.shape}')
 print(f'{df.head(5)}')
 df.to_csv(filename, index=False)
 df.to_csv(filename + '.gz', index=False, compression='gzip' )