Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<data_type_conversions>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import seaborn as sns
import os
from tqdm import tqdm
import gc
import pickle
import scipy
import scipy.signal
for col in('Electrical', 'KitchenQual', 'Exterior1st', 'Exterior2nd', 'Functional', 'MSZoning', 'SaleType', 'Utilities'):
dataset[col] = dataset[col].fillna(dataset[col].mode() [0] )<data_type_conversions>
train = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/train.csv")
 sample_submission = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv" )
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
dataset[col] = dataset[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
dataset[col] = dataset[col].fillna('None' )<data_type_conversions>
train.isna().sum(axis=0 ).sum()
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
dataset[col] = dataset[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
dataset[col] = dataset[col].fillna('None' )<sort_values>
def generate_feature_timedomain() :
def helper(path):
data = []
for file in tqdm(os.listdir(path)) :
tmp = []
file_path = os.path.join(path, file)
 d = pd.read_csv(file_path)
 tmp.append(eval(file[:-4]))
tmp += d.mean(axis=0 ).values.astype('float32' ).tolist()
tmp += d.std(axis=0 ).values.astype('float32' ).tolist()
tmp += d.min(axis=0 ).values.astype('float32' ).tolist()
tmp += d.max(axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.05, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.1, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.2, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.4, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.6, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.8, axis=0 ).values.astype('float32' ).tolist()
for col in d:
d[col+'_5000'] = d[col].shift(5000)
 d[col+'_10000'] = d[col].shift(10000)
 d[col+'_20000'] = d[col].shift(20000)
 d[col+'_30000'] = d[col].shift(30000)
 for col in d.columns[:10]:
col1 = col+'_5000'
col2 = col+'_10000'
col3 = col+'_20000'
col4 = col+'_30000'
tmp1 = d.loc[:, [col, col1]].dropna()
tmp2 = d.loc[:, [col, col2]].dropna()
tmp3 = d.loc[:, [col, col3]].dropna()
tmp4 = d.loc[:, [col, col4]].dropna()
tmp += [tmp1[col].corr(tmp1[col1]),
tmp2[col].corr(tmp2[col2]),
tmp3[col].corr(tmp3[col3]),
tmp4[col].corr(tmp4[col4])]
data.append(tmp)
 return data
print('train_part: ')
 train_part_fea = helper('.. /input/predict-volcanic-eruptions-ingv-oe/train')
 print('test_part: ')
 test_part_fea = helper('.. /input/predict-volcanic-eruptions-ingv-oe/test')
 return train_part_fea, test_part_fea
dataset.isnull().sum() [dataset.isnull().sum() > 0].sort_values(ascending=False )<data_type_conversions>
def generate_feature_freq_domain() :
fs = 100
n = 256
N = 60001
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
def helper(fs, n, N, max_f, delta_f, path):
data = []
for file in tqdm(os.listdir(path)) :
tmp = []
file_path = os.path.join(path, file)
 d = pd.read_csv(file_path)
 tmp.append(eval(file[:-4]))
for i in range(d.shape[1]):
if d.iloc[:, i].isna().sum() > 1000:
tmp += [np.nan] * 7 * 65
tmp += [np.nan] * 10
else:
f, t, Z = scipy.signal.stft(d.iloc[:, i].fillna(0 ).values, fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z_half = np.abs(Z[:round(Z.shape[0]//2)+1] ).T
tmp += Z_half.min(axis=0 ).astype('float32' ).tolist()
tmp += Z_half.max(axis=0 ).astype('float32' ).tolist()
tmp += Z_half.std(axis=0 ).astype('float32' ).tolist()
tmp += Z_half.mean(axis=0 ).astype('float32' ).tolist()
tmp += np.quantile(Z_half, 0.25, axis=0 ).astype('float32' ).tolist()
tmp += np.quantile(Z_half, 0.5, axis=0 ).astype('float32' ).tolist()
tmp += np.quantile(Z_half, 0.75, axis=0 ).astype('float32' ).tolist()
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
tmp += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
data.append(tmp)
 return data
print('train_part: ')
 train_part_fea = helper(fs, n, N, max_f, delta_f, path='.. /input/predict-volcanic-eruptions-ingv-oe/train')
 print('test_part: ')
 test_part_fea = helper(fs, n, N, max_f, delta_f, path='.. /input/predict-volcanic-eruptions-ingv-oe/test')
 return train_part_fea, test_part_fea
dataset['MasVnrType'] = dataset['MasVnrType'].fillna('None')
 dataset['MasVnrArea'] = dataset['MasVnrArea'].fillna(0 )<feature_engineering>
train_part_fea, test_part_fea = generate_feature_timedomain()
dataset["LotFrontage"] = dataset.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<create_dataframe>
with open('train_part_fea_time_domain.pkl', 'wb')as f:
pickle.dump(train_part_fea, f)
 with open('test_part_fea_time_domain.pkl', 'wb')as f:
pickle.dump(test_part_fea, f )
na_percent =(dataset.isnull().sum() /len(dataset)) [(dataset.isnull().sum() /len(dataset)) >0].sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Percentage':na_percent})
 missing_data<data_type_conversions>
train_part_fea_freq, test_part_fea_freq = generate_feature_freq_domain()
dataset['MSSubClass'] = dataset['MSSubClass'].apply(str)
 dataset['YrSold'] = dataset['YrSold'].apply(str)
 dataset['MoSold'] = dataset['MoSold'].apply(str)
 dataset['OverallCond'] = dataset['OverallCond'].astype(str)
 dataset['LotArea'] = dataset['LotArea'].astype(np.int64)
 dataset['MasVnrArea'] = dataset['MasVnrArea'].astype(np.int64 )<feature_engineering>
with open('train_part_fea_freq_domain.pkl', 'wb')as f:
pickle.dump(train_part_fea_freq, f)
 with open('test_part_fea_freq_domain.pkl', 'wb')as f:
pickle.dump(test_part_fea_freq, f )
dataset['TotalSF']=dataset['TotalBsmtSF'] + dataset['1stFlrSF'] + dataset['2ndFlrSF']<feature_engineering>
dataset['TotalBath']=dataset['BsmtFullBath'] + dataset['FullBath'] +(0.5*dataset['BsmtHalfBath'])+(0.5*dataset['HalfBath'] )<feature_engineering>
base_colname = ['sensor_'+str(i)for i in range(1, 11)]
fea_colname = ['segment_id'] + [j + '_mean' for j in base_colname] + [j + '_std' for j in base_colname] + \
[j + '_min' for j in base_colname] + [j + '_max' for j in base_colname] + \
[j + '_5_quant' for j in base_colname] + [j + '_10_quant' for j in base_colname] + \
[j + '_20_quant' for j in base_colname] + [j + '_40_quant' for j in base_colname] + \
[j + '_60_quant' for j in base_colname] + [j + '_80_quant' for j in base_colname] + \
[j + i for j in base_colname for i in ['_5000_self_corr', '_10000_self_corr',
'_20000_self_corr', '_30000_self_corr']]
train = pd.merge(train, pd.DataFrame(train_part_fea, columns=fea_colname), on='segment_id', how='left')
 sample_submission = pd.merge(sample_submission, pd.DataFrame(test_part_fea, columns=fea_colname), on='segment_id', how='left' )
dataset['YrBltAndRemod']=dataset['YearBuilt']+dataset['YearRemodAdd']<feature_engineering>
fea_freq_colname = ['segment_id']
for i in base_colname:
for j in range(65):
for s in ['min','max', 'std', 'mean', '25_quant', '50_quant', '75_quant']:
fea_freq_colname.append(i+'_freq'+str(j)+'_'+s)
 fea_freq_colname.extend([i + ss for ss in ['_A_pow', '_A_num', '_BH_pow', '_BH_num', '_BL_pow',
'_BL_num', '_C_pow', '_C_num', '_D_pow', '_D_num']])
 train = pd.merge(train, pd.DataFrame(train_part_fea_freq, columns=fea_freq_colname), on='segment_id', how='left')
 sample_submission = pd.merge(sample_submission, pd.DataFrame(test_part_fea_freq,
columns=fea_freq_colname), on='segment_id', how='left' )
dataset['Porch_SF'] =(dataset['OpenPorchSF'] + dataset['3SsnPorch'] + dataset['EnclosedPorch'] + dataset['ScreenPorch'] + dataset['WoodDeckSF'] )<feature_engineering>
train.to_pickle('train.pkl')
 sample_submission.to_pickle('test.pkl' )
dataset['Has2ndfloor'] = dataset['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasBsmt'] = dataset['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasFirePlace'] =dataset['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 dataset['Has2ndFlr']=dataset['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasPool']=dataset['PoolArea'].apply(lambda x: 1 if x > 0 else 0 )<categorify>
X_train, X_val, y_train, y_val = train_test_split(train.drop(['segment_id', 'time_to_eruption'], axis=1 ).values,
train['time_to_eruption'].values,
test_size=0.25, random_state=42 )
categorical_col =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for col in categorical_col:
label = LabelEncoder()
label.fit(list(dataset[col].values))
dataset[col] = label.transform(list(dataset[col].values))
print('Shape all_data: {}'.format(dataset.shape))<feature_engineering>
train_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_val, y_val, reference=train_data,)
 params = { 'num_leaves': 85,
'n_estimators': 1000,
'min_data_in_leaf': 10,
'objective':'mae',
'max_depth': -1,
'learning_rate': 0.01,
'max_bins': 2048,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 1,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": -1,
"random_state": 42}
model = lgb.train(params=params, train_set=train_data, valid_sets=[train_data, val_data], valid_names=['train', 'val'],
early_stopping_rounds=50 )
<import_modules><EOS> 
submission = pd.DataFrame({'segment_id': sample_submission['segment_id'].values,
'time_to_eruption': model.predict(sample_submission.iloc[:, 2:].values)})
 submission.to_csv('submission.csv', index=False )