Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<count_values>
import numpy as np
import pandas as pd
from scipy.stats import skew, kurtosis
from scipy.signal import stft
import os
from tqdm.notebook import tqdm
import scipy
import re
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error as mse
import matplotlib.pyplot as plt
from xgboost import plot_importance
import xgboost as xgb
train_df.Pclass.value_counts() /train_df.shape[0]*100<create_dataframe>
train = pd.read_csv('.. /input/predict-volcanic-eruptions-ingv-oe/train.csv')
grid_pivot1 = train_df.pivot_table(columns='Survived', values='Age', aggfunc='mean' )<prepare_x_and_y>
fs = 100
N = 60001
n = 256
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
DIR = '.. /input/predict-volcanic-eruptions-ingv-oe'
<create_dataframe>
def make_features(tgt):
tgt_df = train if tgt == 'train' else test
feature_set = []
for segment_id in tqdm(tgt_df['segment_id']):
segment_df = pd.read_csv(os.path.join(DIR,f'{tgt}/{segment_id}.csv'))
segment = [segment_id]
for sensor in segment_df.columns:
x = segment_df[sensor][:N]
if x.isna().sum() > 1000:
segment +=([np.NaN] * 10)
 continue
f, t, Z = scipy.signal.stft(x.fillna(0), fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
segment += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
feature_set.append(segment)
 cols = ['segment_id']
for i in range(10):
for j in ['A_pow', 'A_num','BH_pow', 'BH_num','BL_pow', 'BL_num','C_pow', 'C_num','D_pow', 'D_num']:
cols += [f's{i+1}_{j}']
feature_df = pd.DataFrame(feature_set, columns = cols)
 feature_df['segment_id'] = feature_df['segment_id'].astype('int')
 return feature_df
grid_pivot2 = train_df.pivot_table(index='Sex', columns='Survived', values='Age', aggfunc='mean')
 grid_pivot2<create_dataframe>
feature_df = make_features('train' )
grid_pivot3 = train_df.pivot_table(index='Pclass',columns='Survived', values='Age' )<create_dataframe>
train_set = pd.merge(train, feature_df, on = 'segment_id' )
train_df.pivot_table(index=['Embarked','Pclass'], columns='Sex', values='Survived' )<define_variables>
def create_columns(df,result):
df = df.fillna(0)
 for column in df:
result.at[index,'sum_'+column] = sum(df[column])
 result.at[index,'med_'+column] = np.median(df[column])
 result.at[index,'permiss_'+column] = df[column].isnull().sum() / df[column].size
result.at[index, 'skew_'+ column] = skew(df[column])
 result.at[index, 'kurtosis'+ column] = kurtosis(df[column])
 result.at[index,'max_'+column] = df[column].max()
result.at[index,'min_'+column] = df[column].min()
result.at[index, 'std_'+ column] = np.std(df[column])
 result.at[index, 'var_'+ column] = np.var(df[column])
 result.at[index,'quan_0.05'+ column] = np.quantile(df[column],0.05)
 result.at[index,'quan_0.1'+ column] = np.quantile(df[column],0.1)
 result.at[index,'quan_0.15'+ column] = np.quantile(df[column],0.15)
 result.at[index,'quan_0.2'+ column] = np.quantile(df[column],0.2)
 result.at[index,'quan_0.025'+ column] = np.quantile(df[column],0.25)
 result.at[index,'quan_0.3'+ column] = np.quantile(df[column],0.3)
 result.at[index,'quan_0.35'+ column] = np.quantile(df[column],0.35)
 result.at[index,'quan_0.4'+ column] = np.quantile(df[column],0.4)
 result.at[index,'quan_0.45'+ column] = np.quantile(df[column],0.45)
 result.at[index,'quan_0.5'+ column] = np.quantile(df[column],0.5)
 result.at[index,'quan_0.55'+ column] = np.quantile(df[column],0.55)
 result.at[index,'quan_0.6'+ column] = np.quantile(df[column],0.6)
 result.at[index,'quan_0.65'+ column] = np.quantile(df[column],0.65)
 result.at[index,'quan_0.7'+ column] = np.quantile(df[column],0.7)
 result.at[index,'quan_0.75'+ column] = np.quantile(df[column],0.75)
 result.at[index,'quan_0.8'+ column] = np.quantile(df[column],0.8)
 result.at[index,'quan_0.85'+ column] = np.quantile(df[column],0.85)
 result.at[index,'quan_0.9'+ column] = np.quantile(df[column],0.9)
 result.at[index,'quan_0.95'+ column] = np.quantile(df[column],0.95)
 return result
def extract_titles(name):
tit = re.findall('([A-Za-z]+)\.', name)
 return tit[0]<feature_engineering>
for row in train_set.itertuples() :
index = row[0]
segmentid = row[1]
if str(segmentid)+".csv" in filenames_train:
df_segement = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/'+str(segmentid)+".csv")
 result = pd.concat([df_segement, df_segement.abs().add_suffix("_abs")], axis=1, join="inner")
 df_segemnt = result
train_set = create_columns(df_segement,train_set)
 if index % 100 == 0:
print(index)
dataset['Title'] = dataset['Name'].apply(lambda x: extract_titles(x))<count_missing_values>
test = pd.DataFrame([(re.findall("[0-9]+",a)) for a in filenames_test] , columns=['segment_id'] )
dataset[dataset['source'] == 'train'].isnull().sum()<count_missing_values>
feature_df_test = make_features('test')
dataset[dataset['source'] == 'test'].isnull().sum()<feature_engineering>
feature_df_test.drop('segment_id', axis = 1, inplace = True )
dataset['Title'] = dataset['Title'].replace(['Don', 'Rev', 'Dr','Major', 'Lady', 'Sir','Col', 'Capt', 'Countess',
'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace(['Ms', 'Mlle'], 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme','Mrs' )<count_values>
test_set = test.join(feature_df_test )
dataset[dataset['source'] == 'train'].Title.value_counts()<drop_column>
for row in test_set.itertuples() :
index = row[0]
segementId = row[1]
if str(segementId)+".csv" in filenames_test:
df_segement = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/test/'+str(segementId)+".csv")
 result = pd.concat([df_segement, df_segement.abs().add_suffix("_abs")], axis=1, join="inner")
 df_segemnt = result
test_set = create_columns(df_segement,test_set)
 if index % 100 == 0:
print(index)
<categorify>
test_set.to_csv('test_work.csv' )
sex_dummy = pd.get_dummies(dataset['Sex'] )<merge>
train_set.to_csv('train_work.csv' )
dataset = dataset.join(sex_dummy )<create_dataframe>
y_train = train_set['time_to_eruption']
train_df = train_set.drop(['time_to_eruption','segment_id'], axis = 1 )
grid_pivot = dataset[dataset['source'] == 'train'].pivot_table(index='Pclass',columns='Sex', values='Age', aggfunc='median' )<find_best_params>
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})" 
.format(results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("" )
guess_ages = np.zeros(( 2,3))
guess_ages
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]<feature_engineering>
param = {'eta': [0.05,0.1,0.2,0.3],
'max_depth': [4,5,6,7,8,9,10],
'subsample ': [0.5,0.75,1],
'gamma': [0.05,0.075,0.09,0.1,0.15,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
}
search = RandomizedSearchCV(xgb.XGBRegressor() ,param,n_iter = 30,cv = 5, scoring = 'neg_mean_squared_error')
 search.fit(train_df,y_train)
dataset['AgeBand'] = pd.cut(dataset['Age'], 5 )<count_values>
params = {'subsample': 0.75, 'max_depth': 9, 'gamma': 0.1, 'eta': 0.1}
dataset.AgeBand.value_counts()<data_type_conversions>
xgb_reg = xgb.XGBRegressor(**params)
 xgb_reg.fit(train_df, y_train )
dataset['AgeBand'] = dataset['AgeBand'].astype(str )<feature_engineering>
sorted_idx = xgb_reg.feature_importances_.argsort()
imp = xgb_reg.feature_importances_
col = train_df.columns
<categorify>
todrop = col[sorted_idx[:50]]
def ageclass(x):
if x <= 16:
return 0
elif x > 16 and x <= 32:
return 1
elif x > 32 and x <= 48:
return 2
elif x > 48 and x <= 64:
return 3
else:
x > 64
return 4<feature_engineering>
train_df.drop(todrop, axis = 1, inplace = True)
 test_set.drop(todrop, axis = 1, inplace = True )
dataset['AgeClass'] = dataset['Age'].apply(ageclass )<feature_engineering>
xgb_reg = xgb.XGBRegressor(**params)
 xgb_reg.fit(train_df, y_train )
dataset['Family_Size'] = dataset['Parch'] + dataset['SibSp'] + 1<create_dataframe>
test_df = test_set.drop('segment_id', axis = 1 )
dataset.pivot_table(index='Family_Size', values='Survived' ).sort_values(by='Survived',ascending=False )<compute_test_metric>
submission = pd.DataFrame(test_set['segment_id'], columns=['segment_id'] )
def isalone(x):
if x == 1:
return 1
else:
return 0<feature_engineering>
submission['time_to_eruption'] = xgb_reg.predict(test_df )
dataset['IsAlone'] = dataset['Family_Size'].apply(isalone )<data_type_conversions>
submission.set_index('segment_id', inplace = True )
dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace=True )<count_missing_values>
submission.to_csv('out.csv' )