Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<groupby>
ver = 27
train_grouped = train_df.groupby(['Sex','Title','Pclass'])
 train_grouped.Age.mean()<feature_engineering>
plt.style.use('ggplot')
 def get_df_name(df):
name =[x for x in globals() if globals() [x] is df][0]
return name
def tstats(t, doplot = False):
print('
for _,i in enumerate(t):
nulls = t[i].isna().sum()
if nulls > 0:
print(i,'=', t[i].nunique() , ',NULLS = ', nulls, ',% of nulls = ',round(100*nulls/t.shape[0]))
else:
print(i,'=', t[i].nunique())
 if doplot:
print()
print(t.sample(10))
print()
warnings.filterwarnings('ignore' )
train_df.Age = train_grouped.Age.apply(lambda x: x.fillna(x.mean()))<count_missing_values>
homedir = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'
def read_n_files(dirname, nfiles):
tmp = pd.DataFrame()
cnt = 0
for dirname, _, filenames in os.walk(homedir + dirname):
for filename in filenames:
cnt += 1
if cnt > nfiles: break
t = pd.read_csv(os.path.join(dirname, filename))
t['segment_id'] = filename[:-4]
tmp = pd.concat([tmp, t], ignore_index=True)
 return tmp
t10 = read_n_files('train', 10 ).fillna(0)
 float_cols = [c for c in t10 if t10[c].dtype == "float64"]
float32_cols = {c: np.float32 for c in float_cols}
nparts = 6
r = [int(j*30000/nparts)for j in range(nparts+1)]
def get_peaks(tab, col):
N = tab.shape[0]
T = 1.0 / 800.0
x = np.linspace(0.0, N*T, N)
 y = tab[col].values
yf = scipy.fftpack.fft(y)
 xf = np.linspace(0.0, int(1.0/(2.0*T)) , int(N/2))
x = np.log(2.0/N * np.abs(yf[:N//2]))
peaks, _ = find_peaks(x, distance=500)
 return x, peaks
def read_n_files32(dirname, nfiles):
tmp_res = pd.DataFrame()
cnt = 0
for dirname, _, filenames in os.walk(homedir + dirname):
for filename in tqdm_notebook(filenames):
cnt += 1
if cnt > nfiles: break
t = pd.read_csv(os.path.join(dirname, filename), engine='c', dtype=float32_cols)
 res = []
for i in [i for i in range(1,11)]:
x, peaks = get_peaks(t, 'sensor_'+str(i))
for j in range(len(r)-1):
xpeaks = peaks[(peaks > r[j])&(peaks <= r[j+1])]
ypeaks = x[xpeaks]
mx = round(np.median(xpeaks))
sx = round(np.std(xpeaks))
my = round(np.median(ypeaks), 2)
 sy = round(np.std(ypeaks), 2)
 r0, r1 = r[j], r[j+1]
res.append([f's{i:02d}_r{r0:05d}_{r1}_mx', mx])
 res.append([f's{i:02d}_r{r0:05d}_{r1}_sx', sx])
 res.append([f's{i:02d}_r{r0:05d}_{r1}_my', my])
 res.append([f's{i:02d}_r{r0:05d}_{r1}_sy', sy])
 df1 = pd.DataFrame({'ind':peaks, 'x':x[peaks]})
 df2 = pd.DataFrame({'ind':np.linspace(0,30000,30001 ).astype(int)})
 df = df2.merge(df1, how='left', on='ind')
 df = df.interpolate(method='linear', limit_direction='both', axis=0)
 data = df['x'].values
data[0] = data[:5000].min()
peaks, _ = find_peaks(data)
 prominences = peak_prominences(data, peaks)[0]
contour_heights = data[peaks] - prominences
results_half = peak_widths(data, peaks, rel_height=0.5)
 results_half[0]
results_full = peak_widths(data, peaks, rel_height=1)
 results_full[0]
peak_data = pd.DataFrame({
'peak_x':peaks,
'peak_y':data[peaks],
'width':results_full[0],
'height':prominences
} ).sort_values(by='height', ascending=False)
 tmp = peak_data.reset_index(drop=True ).reset_index().head(5)
 tmp['index'] = tmp['index'].astype(str ).apply(lambda x: x.zfill(2))
tmp = tmp.melt(id_vars=['index'], value_vars=['peak_x', 'peak_y', 'width', 'height'])
 tmp['variable'] = tmp['variable'] + tmp['index']
tmp = np.array(tmp[['variable', 'value']])
 for titem in tmp:
res.append([f's{i:02d}p_' + titem[0], titem[1]])
 t = pd.DataFrame(np.array(res), columns=['col', 'val'])
 t['segment_id'] = filename[:-4]
t['val'] = t['val'].astype(float)
 t = t.fillna(0)
 t = t.pivot_table(
 index='segment_id',
columns='col',
values='val'
).reset_index()
tmp_res = pd.concat([tmp_res, t], ignore_index=True)
 return tmp_res.fillna(0)
 def read_all_files(dirname):
tmp = pd.DataFrame()
cnt = 0
for dirname, _, filenames in os.walk(homedir + dirname):
for filename in filenames:
cnt += 1
t = pd.read_csv(os.path.join(dirname, filename)).fillna(0)
 cols = t.columns
t['segment_id'] = filename[:-4]
t = t.groupby('segment_id', as_index=False ).median()
tmp = pd.concat([tmp, t], ignore_index=True)
 return tmp
train_df.Age.isnull().sum()<feature_engineering>
tmp = read_n_files32('train', 10 )
test_df['Title'] = test_df.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 test_df.Title = test_df.Title.map(norm_titles)
 test_grouped = test_df.groupby(['Sex','Title','Pclass'])
 test_df.Age = test_grouped.Age.apply(lambda x: x.fillna(x.mean()))
test_df.Age.isnull().sum()
test_df.Title.value_counts()<feature_engineering>
tstats(tmp,1 )
most_embarked = train_df.Embarked.value_counts().index[0]
train_df.Embarked = train_df.Embarked.fillna(most_embarked)
 train_df.Fare = train_df.Fare.fillna(train_df.Fare.median())
 train_df.Cabin = train_df.Cabin.fillna('U')
 train_df.Cabin = train_df.Cabin.map(lambda x: x[0])
 train_df['Cabin'] = train_df.Cabin.replace({'T': 'G'})
 train_df.Cabin.value_counts() 
<feature_engineering>
test_df.Cabin = test_df.Cabin.fillna('U')
 most_embarked = test_df.Embarked.value_counts().index[0]
test_df.Embarked = test_df.Embarked.fillna(most_embarked)
 test_df.Fare = test_df.Fare.fillna(train_df.Fare.median())
 test_df['Cabin'] = test_df.Cabin.apply(lambda name: name[0])
 test_df.Cabin.value_counts()<categorify>
def prepare_data() :
t4m = pd.read_csv('/kaggle/input/volcanic-eruptions/t4m_10.csv')
 t4p = pd.read_csv('/kaggle/input/volcanic-eruptions/t4p_10.csv')
 features = t4m.columns[1:-1]
target = 'time_to_eruption'
df = t4m[t4m.columns[1:]].copy()
t = df.corr() [target].reset_index()
t[target] = abs(t[target])
 t.sort_values(by=target, ascending = False)
 t = t.loc[(t[target] > 0.05)&(t[target] < 1)].sort_values(by=target, ascending=False ).reset_index()
good_features = t['index'].values
scaler = StandardScaler()
t = pd.DataFrame(scaler.fit_transform(t4m[good_features]), columns = good_features)
 t['sum_feat'] = t[good_features].sum(axis=1)
 t['mul_feat'] = t[good_features].prod(axis=1)
 t['s_m_feat'] = t['sum_feat'] * t['mul_feat']
t4m['sum_feat'] = t['sum_feat'].values
t4m['mul_feat'] = t['mul_feat'].values
t4m['s_m_feat'] = t['s_m_feat'].values
t = pd.DataFrame(scaler.fit_transform(t4p[good_features]), columns = good_features)
 t['sum_feat'] = t[good_features].sum(axis=1)
 t['mul_feat'] = t[good_features].prod(axis=1)
 t['s_m_feat'] = t['sum_feat'] * t['mul_feat']
t4p['sum_feat'] = t['sum_feat'].values
t4p['mul_feat'] = t['mul_feat'].values
t4p['s_m_feat'] = t['s_m_feat'].values
feat = list(features)
 feat.append('sum_feat')
 feat.append('mul_feat')
 feat.append('s_m_feat')
 X, y = t4m[feat], t4m[target]
data = y.values.reshape(-1, 1)
 rng = np.random.RandomState(304)
 qt = QuantileTransformer(n_quantiles=1000, output_distribution='normal',
random_state=rng)
 z = qt.fit_transform(data)
 z_back = qt.inverse_transform(z)
 plt.figure(figsize=(15,5))
plt.subplot(131)
 plt.hist(data)
 plt.title('y')
 plt.subplot(132)
 plt.hist(z, bins = 50)
 plt.title('z')
 plt.subplot(133)
 plt.hist(z_back, bins = 50)
 plt.title('inverse')
 plt.show()
return t4m, t4p, X, y, z, feat, target, qt
t4m, t4p, X, y, z, feat, target, qt = prepare_data()
def de_qt(val):
return qt.inverse_transform(val.reshape(-1, 1)).ravel()
train_df.Sex = train_df.Sex.map({"male": 0, "female":1})
 pclass_dummies = pd.get_dummies(train_df.Pclass, prefix="Pclass")
 title_dummies = pd.get_dummies(train_df.Title, prefix="Title")
 cabin_dummies = pd.get_dummies(train_df.Cabin, prefix="Cabin")
 embarked_dummies = pd.get_dummies(train_df.Embarked, prefix="Embarked")
 train_dummies = pd.concat([train_df, pclass_dummies, title_dummies, cabin_dummies, embarked_dummies], axis=1)
 train_dummies.drop(['Pclass', 'Title', 'Cabin', 'Embarked', 'Name', 'Ticket'], axis=1, inplace=True)
 train_dummies.head()<categorify>
%%time
scaler = StandardScaler()
nval = 1000
ntest = 500
t = X.copy()
t[target] = z
t = t.sample(frac=1)
 t_init = t.copy()
t[feat] = scaler.fit_transform(t[feat])
 t1 = t[:nval]
t2 = t[nval:nval+ntest]
t3 = t[nval+ntest:]
X_train, y_train = t1[feat], t1[target]
X_val, y_val = t2[feat], t2[target]
X_test, y_test = t3[feat], t3[target]
model = Sequential()
model.add(Dense(128, input_shape=(X.shape[1],), activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(16, activation='relu', kernel_regularizer= kreg.l1(0.01)))
 model.add(Dense(1))
model.compile(loss='mse', optimizer='rmsprop', metrics=['mae'])
 history = model.fit(X_train, y_train, epochs=70, batch_size=4, verbose=False,
validation_data =(X_val, y_val))
plot_history(history, 'mae')
 t = t_init.copy()
t1 = t[:nval]
t2 = t[nval:nval+ntest]
t3 = t[nval+ntest:]
X_train, y_train = t1[feat], t1[target]
X_val, y_val = t2[feat], t2[target]
X_test, y_test = t3[feat], t3[target]
X_train['keras'] = model.predict(X_train)
 X_test['keras'] = model.predict(X_test)
 model = CatBoostRegressor(verbose=False)
 y_pred = de_qt(model.fit(X_train[feat], y_train ).predict(X_test[feat]))
y_true = de_qt(y_test.values)
 lmetric = int(mean_absolute_error(y_true, y_pred))
print('The MAE of prediction w/o keras is:', lmetric)
 model = CatBoostRegressor(verbose=False)
 y_pred = de_qt(model.fit(X_train, y_train ).predict(X_test))
y_true = de_qt(y_test.values)
 lmetric = int(mean_absolute_error(y_true, y_pred))
print('The MAE of prediction with keras is:', lmetric)
 importances = model.feature_importances_
target_results = pd.DataFrame(importances, columns=[target])
 target_results['feature'] = X_train.columns
t = target_results.sort_values(by = target)
 t = t.tail(15)
 plt.figure(figsize=(5, 5))
plt.title('Feature Importance')
 plt.barh(t['feature'], t[target])
 plt.show()
<import_modules><EOS> 
%%time
n_fold = 7
folds = KFold(n_splits=n_fold, shuffle=True, random_state=101)
 params = {
"n_estimators": 2000,
"boosting_type": "gbdt",
"metric": "mae",
"num_leaves": 66,
"learning_rate": 0.005,
"feature_fraction": 0.9,
"bagging_fraction": 0.8,
"agging_freq": 3,
"max_bins": 2048,
"verbose": 0,
"random_state": 101,
"nthread": -1,
"device": "cpu",
}
y_pred = np.zeros(t4p.shape[0])
 for n_fold,(trn_idx, val_idx)in enumerate(folds.split(t4m)) :
print(f"Fold {n_fold}:")
 trn_x, trn_y = t4m[feat].iloc[trn_idx], t4m[target].iloc[trn_idx]
val_x, val_y = t4m[feat].iloc[val_idx], t4m[target].iloc[val_idx]
model = lgb.LGBMRegressor(**params)
 model.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric="mae", verbose=200, early_stopping_rounds=50 
)
 y_pred += model.predict(t4p[feat], num_iteration=model.best_iteration_)/ folds.n_splits
plt.figure()
plt.hist(y_pred, bins=50)
 plt.title('y pred distribution')
 plt.show()
my_submission = pd.DataFrame({
'segment_id':t4p['segment_id'].values,
'time_to_eruption':y_pred
})
 my_submission.loc[my_submission[target] < 60000, target] = 60000
my_submission.loc[my_submission[target] > 4.8e+07, target] = 4.8e+07
my_submission.to_csv(f'submission_{ver}.csv', index=False )