Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<feature_engineering>
import os
from pathlib import Path
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
from sklearn.base import TransformerMixin
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.inspection import permutation_importance
from sklearn.feature_selection import chi2, SelectKBest
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_absolute_error
import optuna
from xgboost import XGBRegressor
skew_features = [
'LotFrontage', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'MiscVal',
'TotalSF', 'TotalPorchSF',
]
for feature in skew_features:
df[feature] = boxcox1p(df[feature], boxcox_normmax(df[feature] + 1))<define_variables>
data_folder = Path('.. /input/predict-volcanic-eruptions-ingv-oe/')
 df = pd.read_csv(data_folder / 'train.csv')
 df.head().T
exterQual = {'Fa': 1, 'TA': 2, 'Gd': 3,'Ex': 4}
exterCond = {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
bsmtQual = {'None': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
bsmtCond = {'None': 0, 'Po': 1, 'Fa': 2,'TA': 3, 'Gd': 4}
bsmtExposure = {'None': 0, 'No': 1, 'Mn': 2,'Av': 3, 'Gd': 4}
bsmtFinType1 = {'None': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6}
bsmtFinType2 = {'None': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6}
heatingQC = {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
kitchenQual = {'Fa': 0, 'TA': 1, 'Gd': 2, 'Ex': 3}
firePlaceQu = {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
garageQual = {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
garageCond = {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
functional = {'Typ': 0, 'Min1': 1, 'Min2': 1, 'Mod': 2, 'Maj1': 3, 'Maj2': 3, 'Sev': 4}
landSlope = { 'Sev': 1, 'Mod': 2, 'Gtl': 3 }
df['ExterQual'] = df['ExterQual'].map(exterQual)
 df['ExterCond'] = df['ExterCond'].map(exterCond)
 df['BsmtQual'] = df['BsmtQual'].map(bsmtQual)
 df['BsmtCond'] = df['BsmtCond'].map(bsmtCond)
 df['BsmtExposure'] = df['BsmtExposure'].map(bsmtExposure)
 df['BsmtFinType1'] = df['BsmtFinType1'].map(bsmtFinType1)
 df['BsmtFinType2'] = df['BsmtFinType2'].map(bsmtFinType2)
 df['HeatingQC'] = df['HeatingQC'].map(heatingQC)
 df['KitchenQual'] = df['KitchenQual'].map(kitchenQual)
 df['FireplaceQu'] = df['FireplaceQu'].map(firePlaceQu)
 df['GarageQual'] = df['GarageQual'].map(garageQual)
 df['GarageCond'] = df['GarageCond'].map(garageCond)
 df['Functional'] = df['Functional'].map(functional)
 df['LandSlope'] = df['LandSlope'].map(landSlope )<drop_column>
df = df.sort_values('time_to_eruption')
 time_min = df.head(1 ).iloc[0]['segment_id']
time_max = df.tail(1 ).iloc[0]['segment_id']
df_min = pd.read_csv(data_folder / 'train' / f'{time_min}.csv')
 df_max = pd.read_csv(data_folder / 'train' / f'{time_max}.csv')
 df_min['time_to_eruption'] = df.head(1 ).iloc[0]['time_to_eruption']
df_max['time_to_eruption'] = df.tail(1 ).iloc[0]['time_to_eruption']
df.drop(columns=['Street', 'Utilities', 'Fireplaces',
'PoolArea', 'PoolQC', 'GarageYrBlt',
'GarageArea', 'TotalBsmtSF', '1stFlrSF',
'FullBath', 'YearBuilt', 'YearRemodAdd',
], inplace=True )<categorify>
for i in range(10):
sensor = f'sensor_{i + 1}'
if df_min[sensor].isnull().all() or df_max[sensor].isnull().all() :
del df_min[sensor]
del df_max[sensor]
df = pd.get_dummies(df,columns=df.select_dtypes(exclude=np.number ).columns.tolist())
 df.shape<filter>
from tsfresh import extract_features
from tsfresh.feature_extraction import MinimalFCParameters
print(train[np.logical_and(train['OverallQual'] < 5, train['SalePrice'] > 200000)].index)
 print(train[np.logical_and(train['GrLivArea'] > 4000, train['SalePrice'] < 300000)].index)
 print(df[np.logical_and(df['OverallQual']==4, df['SalePrice'] > 200000)].index)
 outliers=[457, 523, 1298]<count_values>
tsfresh_parameters = MinimalFCParameters()
sparse=[]
for feature in X_test.columns:
counts = X_test[feature].value_counts()
zeros = counts.iloc[0]
if zeros/len(X_test)> 99.94:
sparse.append(feature)
 print(feature)
 X_test.drop(columns=sparse, inplace=True)
 X_train.drop(columns=sparse, inplace=True )<prepare_x_and_y>
del tsfresh_parameters['length']
tsfresh_parameters['skewness'] = None
tsfresh_parameters['kurtosis'] = None
tsfresh_parameters['last_location_of_maximum'] = None
tsfresh_parameters['first_location_of_maximum'] = None
tsfresh_parameters['last_location_of_minimum'] = None
tsfresh_parameters['first_location_of_minimum'] = None
tsfresh_parameters['first_location_of_minimum'] = None
tsfresh_parameters['benford_correlation'] = None
tsfresh_parameters['percentage_of_reoccurring_values_to_all_values'] = None
tsfresh_parameters['percentage_of_reoccurring_datapoints_to_all_datapoints'] = None
tsfresh_parameters['number_peaks'] = [
{'n': 1},
{'n': 3},
{'n': 5},
{'n': 10},
{'n': 50}
]
tsfresh_parameters['binned_entropy'] = [
{'max_bins': 10}
]
tsfresh_parameters['fft_aggregated'] = [
{'aggtype': 'centroid'},
{'aggtype': 'variance'},
{'aggtype': 'skew'},
{'aggtype': 'kurtosis'}
]
tsfresh_parameters['autocorrelation'] = [
{'lag': 0},
{'lag': 1},
{'lag': 2},
{'lag': 3},
{'lag': 4},
{'lag': 5},
{'lag': 6},
{'lag': 7},
{'lag': 8},
{'lag': 9}
]
tsfresh_parameters['agg_autocorrelation'] = [
{'f_agg': 'mean', 'maxlag': 40},
{'f_agg': 'median', 'maxlag': 40},
{'f_agg': 'var', 'maxlag': 40}
]
tsfresh_parameters['friedrich_coefficients'] = [
{'coeff': 0, 'm': 3, 'r': 30},
{'coeff': 1, 'm': 3, 'r': 30},
{'coeff': 2, 'm': 3, 'r': 30},
{'coeff': 3, 'm': 3, 'r': 30}
]
tsfresh_parameters['count_above'] = [{'t': 0}]
tsfresh_parameters['count_below'] = [{'t': 0}]
X_train = X_train.drop(X_train.index[outliers])
 Y_train = Y_train.drop(Y_train.index[outliers])
 y_train = Y_train.apply(np.log1p )<normalization>
def preprocess_timeseries(data, parameters, is_train=True):
df_result = None
if is_train:
segments = df.iterrows()
else:
segments = enumerate(os.listdir(data_folder / 'test'))
for idx, row in segments:
if is_train:
segment, time_to_eruption = row
segment_timeseries_path = data_folder / 'train/{segment}.csv'
else:
segment = row
segment_timeseries_path = data_folder / 'test/{segment}'
segment_timeseries = pd.read_csv(segment_timeseries_path)
 segment_timeseries = segment_timeseries.fillna(0 ).reset_index()
segment_timeseries['id'] = idx
extracted_features = extract_features(segment_timeseries, \
column_id='id', \
column_sort='index', \
disable_progressbar=True,
default_fc_parameters=parameters)
 extracted_features['segment'] = segment
if is_train:
extracted_features['time_to_eruption'] = time_to_eruption
if df_result is None:
df_result = extracted_features
else:
df_result = pd.concat([df_result, extracted_features], \
axis=0, \
ignore_index=True, \
sort=True)
 print(f'Processed segment
return df_result
scaler = RobustScaler()
x_train = pd.DataFrame(scaler.fit_transform(X_train),
index=X_train.index,
columns=X_train.columns)
 X_test = pd.DataFrame(scaler.transform(X_test),
index=X_test.index,
columns=X_test.columns )<train_on_grid>
def save(parameters):
ts_train = pd.read_csv(data_folder / 'train.csv')
 df_train = preprocess_timeseries(ts_train, parameters)
 df_train.to_csv('train.csv', index=False)
 df_test = preprocess_timeseries(None, parameters, is_train=False)
 df_test.to_csv('test.csv', index=False )
kfolds = KFold(n_splits=8, shuffle=True, random_state=42)
 def cv_rmse(model, X=x_train, y=y_train):
rmse = np.sqrt(-cross_val_score(model, X, y,scoring="neg_mean_squared_error",cv=kfolds))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def fit_model(model, X=x_train, y=y_train):
return model.fit(X, y )<train_on_grid>
df_train = pd.read_csv('.. /input/volcanoes-ts-processed-with-tsfresh/train.csv')
 df_train = df_train.dropna(axis='columns')
 features = [c for c in df_train.columns if c not in ['time_to_eruption', 'segment']]
X = df_train[features]
y = df_train['time_to_eruption']
df_train.head()
def grid_search(model, param_grid, cv=5,
scoring='neg_mean_squared_error'):
grid_model = GridSearchCV(model,
param_grid,
cv=cv,
scoring=scoring,
return_train_score=True 
)
 fit_model(grid_model)
 cvres = grid_model.cv_results_
for mean_score, std_score, params in zip(cvres["mean_test_score"], cvres["std_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), std_score, params)
 return grid_model<define_search_space>
class LowImportanceSelector(TransformerMixin):
def __init__(self, threshold, n_estimators=100):
self.features = None
self.threshold = threshold
self.n_estimators = n_estimators
def fit(self, X, y):
estimator = RandomForestRegressor(n_estimators=self.n_estimators)
 estimator.fit(X, y)
 importances = pd.DataFrame({
'feature': X.columns,
'importance': estimator.feature_importances_
})
 importances = importances[importances['importance'] > self.threshold]
self.features = importances['feature']
return self
def transform(self, X):
return X[self.features]
param_grid=[{'alpha': [0.0005, 0.0007]}]
lasso = Lasso(max_iter=1e7, random_state=SEED)
 grid_lasso = grid_search(lasso, param_grid, cv=kfolds )<train_on_grid>
class CorrelationSelector(TransformerMixin):
def __init__(self, threshold):
self.columns = None
self.threshold = threshold
def fit(self, X, y=None):
X = X.copy()
self.columns = set()
C = X.corr()
for i in range(len(C.columns)) :
for j in range(i):
if(C.iloc[i, j] >= self.threshold)and(C.columns[j] not in self.columns):
c = C.columns[i]
self.columns.add(c)
 if c in X.columns:
del X[c]
return self
def transform(self, X, y=None):
X.drop(columns=list(self.columns)).shape
return X.drop(columns=list(self.columns))
param_grid=[
{
'alpha': [ 0.008],
'l1_ratio': [0.0056, 0.009, 0.01, 0.05],
}
]
elastic = ElasticNet(max_iter=1e7, random_state=SEED)
 grid_elastic = grid_search(elastic, param_grid, cv=kfolds )<train_on_grid>
def objective(trial, data=X, target=y):
parameters = {
'tree_method': 'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4, 0.5, 0.6, 0.7, 0.8, 1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008, 0.009, 0.01, 0.012, 0.014, 0.016, 0.018, 0.02]),
'n_estimators': 1000,
'max_depth': trial.suggest_categorical('max_depth', [5, 7, 9, 11, 13, 15, 17, 20]),
'random_state': trial.suggest_categorical('random_state', [24, 48, 2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
}
X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state=1337)
 model = XGBRegressor(**parameters)
 model.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=100, verbose=False)
 return mean_absolute_error(y_test, model.predict(X_test))
param_grid=[
{
'C': np.logspace(1.5,2,4),
'epsilon': [0.05, 0.06],
'gamma':[0.0001]
}
]
svr= SVR()
grid_svr = grid_search(svr, param_grid, cv=kfolds )<choose_model_class>
parameters = {
'lambda': 0.0020555245431348778,
'alpha': 0.11298627316540845,
'colsample_bytree': 0.6,
'subsample': 1.0,
'learning_rate': 0.01,
'max_depth': 20,
'random_state': 48,
'min_child_weight': 18
}
param_grid=[
{
'learning_rate': [0.01],
'max_depth':[ 4 ],
}
]
gbr = GradientBoostingRegressor(
 n_estimators=3000, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=SEED)
 grid_gbr = grid_search(gbr, param_grid, cv=kfolds )<choose_model_class>
pipe = Pipeline([ 
('correlation', CorrelationSelector(threshold=0.85)) , 
('scaler', MinMaxScaler()), 
('xgboost', XGBRegressor(objective='reg:squarederror', n_estimators=1000, **parameters))
] )
param_grid=[
{
'learning_rate': [0.005, 0.009, 0.01],
'n_estimators': [5000]
}
]
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
random_state=SEED,verbose=-1 
)
 grid_lgb = grid_search(lightgbm, param_grid, cv=kfolds )<train_on_grid>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1337 )
param_grid=[
{
'learning_rate': [0.1],
'n_estimators': [3460],
'max_depth':[ 3 ],
'gamma':[0.001],
'subsample':[0.7]
}
]
xgb = XGBRegressor(n_jobs=3)
 grid_xgb = grid_search(xgb, param_grid, cv=kfolds )<train_model>
pipe.fit(X_train, y_train )
stacking = StackingRegressor(estimators=[ 
('ridge', grid_ridge.best_estimator_), 
('lasso', grid_lasso.best_estimator_), 
('elastic',grid_elastic.best_estimator_), 
('xgb', grid_xgb.best_estimator_), 
('lgb', grid_lgb.best_estimator_)
 ],
final_estimator=grid_xgb.best_estimator_,
passthrough=True)
 stacking_model=fit_model(stacking)
 loss=rmsle(y_train, stacking_model.predict(x_train))
print("Stacking:
Loss: %.4f"%(loss))<predict_on_test>
pipe.score(X_test, y_test )
def blend(x):
return(
 0.1*grid_elastic.best_estimator_.predict(x)+
0.05*grid_lasso.best_estimator_.predict(x)+
0.1*grid_ridge.best_estimator_.predict(x)+
0.1*grid_svr.best_estimator_.predict(x)+
0.15*grid_gbr.best_estimator_.predict(x)+
0.1*grid_lgb.best_estimator_.predict(x)+
0.1*grid_xgb.best_estimator_.predict(x)+
0.3*stacking.predict(x)
)
 y_pred = blend(x_train)
 scores=rmsle(y_train, y_pred)
 print("Loss:
%.4f"%(scores))
Y_test = blend(X_test )<save_to_csv>
pipe.fit(X, y )
Y_test = np.expm1(Y_test)
 def save_csv(y):
submission = pd.DataFrame()
submission['Id'] = test_ids
submission['SalePrice'] = y
submission.to_csv('submission.csv',index=False)
 save_csv(Y_test )<load_from_csv>
df_predict = pd.read_csv('.. /input/volcanoes-ts-processed-with-tsfresh/predict.csv')
 df_predict = df_predict.dropna(axis='columns')
 df_predict['segment'] = df_predict['segment'].apply(lambda s: s.split('.')[0] )
<feature_engineering><EOS> 
target = pipe.predict(df_predict.drop(columns='segment'))
target = pd.DataFrame({
'segment_id': df_predict['segment'], 'time_to_eruption': target
})
 target.to_csv('submission.csv', index=False )