Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<filter>
import tensorflow as tf
from tensorflow.keras import Model, Input, models
from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, ReduceLROnPlateau
from tensorflow.keras.layers import BatchNormalization, Dropout, Flatten, Dense, InputLayer, Concatenate, Add, SeparableConv2D, Layer
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.initializers import TruncatedNormal
from sklearn.model_selection import RepeatedKFold
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
train_cleaned = dataset[dataset['source'] == 'train']
test_cleaned = dataset[dataset['source'] == 'test']<data_type_conversions>
train = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')
 sample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')
 train_ids, test_ids = train['segment_id'], sample_submission['segment_id']
y = train['time_to_eruption']
X = np.load('/kaggle/input/volcano-pca/train.npy')
 X_test = np.load('/kaggle/input/volcano-pca/test.npy')
 X_rolling = np.load('/kaggle/input/volcano-pca/rolling_train.npy')
 X_rolling_test = np.load('/kaggle/input/volcano-pca/rolling_test.npy')
 X_concat = np.load('/kaggle/input/volcano-pca/train_concat.npy')
 X_aug = np.load('/kaggle/input/volcano-pca/aug_train_concat_2.npy')
 y_aug = np.load('/kaggle/input/volcano-pca/y_aug2.npy')
 X = X_aug[:, :, :16]
X_rolling = X_aug[:, :, 16:]
MODEL_NAME = 'res32_v3'
In1 = Input(shape=(10, 16, 1))
In2 = Input(shape=(10, 16, 1))
Inm = Input(shape=(10, 32, 1))
train_cleaned['Survived'] = train_cleaned['Survived'].astype(int )<predict_on_test>
WInit = TruncatedNormal(0, 1, 111)
 def concat_inputs(Inp1, Inp2):

 return Concatenate(axis=2 )([Inp1, Inp2])
 def fc(Inp, unit=100, n=3, activation='relu', pred=True, bn=False):

 Inp = Flatten()(Inp)
 out = Dense(unit, activation=activation ,kernel_initializer=WInit )(Inp)
 for i in range(0, n-2):
out = Dense(unit, activation=activation ,kernel_initializer=WInit )(out)
 if bn: out = BatchNormalization()(out)
 if not pred:
out = Dense(unit, activation=activation ,kernel_initializer=WInit )(out)
 return out
out = Dense(1, kernel_initializer=TruncatedNormal(10, 1e-1, 11))(out)
 return out
def submit_prediction(pred, write=True, file_name="", Return=True):

 pred = np.where(pred < 0, np.mean(pred), pred)
 sample_submission['time_to_eruption'] = pred
if write: sample_submission.to_csv(f'{file_name}.csv', index=False)
 if Return: return pred
def vertical_kernel(x,y, length): return [(y, x + i)for i in range(length)]
def horizontal_kernel(x,y, length): return [(y + i, x)for i in range(length)]
def module(inp, vertical_kernels, horizontal_kernels, unit=16, std=1e-1, seed=11):
vertical_convs = [SeparableConv2D(unit,
kernel_size,
kernel_initializer=TruncatedNormal(0, std, seed),
bias_initializer=TruncatedNormal(0, std, seed))(inp)for kernel_size in vertical_kernels]
horizontal_convs = [SeparableConv2D(unit,
kernel_size,
kernel_initializer=TruncatedNormal(0, std, seed),
bias_initializer=TruncatedNormal(0, std, seed))(inp)for kernel_size in horizontal_kernels]
vertical_concats = [Concatenate(axis=2 )(
 [vertical_convs[i], vertical_convs[len(vertical_convs)- 1 - i]] 
)for i in range(len(vertical_convs)// 2)]
horizontal_concats = [Concatenate(axis=1 )(
 [horizontal_convs[i], horizontal_convs[len(horizontal_convs)- 1 - i]] 
)for i in range(len(horizontal_convs)// 2)]
return vertical_concats, horizontal_concats
def predict_model(dtrain, dtest, predictor, outcome, model):
model.fit(dtrain[predictor], dtrain[outcome])
 dtrain_pred = model.predict(dtest[predictor])
 score = model.score(dtrain[predictor], dtrain[outcome])*100
return score, dtrain_pred<define_variables>
def model2(inp):
vertical_kernels = vertical_kernel(6,10, 8)
 horizontal_kernels = horizontal_kernel(32, 3, 8)
 vert_concat1, hor_concat1 = module(inp, vertical_kernels, horizontal_kernels)
 vert_preds = [fc(layer, unit=100, n=5, activation='relu', pred=True)for layer in vert_concat1]
hor_preds = [fc(layer, unit=100, n=5, activation='relu', pred=True)for layer in hor_concat1]
vert_preds.extend(hor_preds)
 pred = fc(Concatenate()(vert_preds), unit=25, n=3, activation='relu', pred=True)
 return pred
predictors_var = ['Pclass','Title', 'female','male', 'AgeClass', 'IsAlone', 'C', 'Q', 'S', 'FareClass', 'Age*Class']
outcome_var = 'Survived'
traindf = train_cleaned
testdf = test_cleaned<choose_model_class>
def resnet_module(inp, unit=32, stride1=(3,3), stride2=(5,5), stride3=(3,3), mid_activation=None, padding='same', bn=True, std=1e-2, seed=123):
l1 = SeparableConv2D(filters=unit,
kernel_size=stride1,
kernel_initializer=TruncatedNormal(0, std, seed),
padding=padding
)(inp)
 l2 = SeparableConv2D(filters=unit,
kernel_size=stride2,
kernel_initializer=TruncatedNormal(0, std*1.1, seed),
activation=mid_activation,
padding='same'
)(l1)
 l3 = SeparableConv2D(filters=unit,
kernel_size=stride3,
kernel_initializer=TruncatedNormal(0, std*5, seed),
padding=padding
)(l2)
 la = Add()([l1, l3])
 if bn: return BatchNormalization()(la)
 return la
def mres(inp):
b1 = resnet_module(inp, stride1=(1,3), stride2=(5,5), stride3=(3,1), bn=False, std=1)
 b1 = resnet_module(b1, stride1=(3,1), stride2=(3,3), stride3=(1,3), mid_activation='relu',bn=True, std=1.5)
 b1 = resnet_module(b1, stride1=(3,3), stride2=(3,3), stride3=(3,3), mid_activation='relu',bn=True, std=1)
 b2 = resnet_module(b1, unit=64, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='relu', bn=False, std=1e-2)
 b2 = resnet_module(b2, unit=64, stride1=(6,1), stride2=(3,3), stride3=(1,6),bn=False, std=1.5e-3)
 b2 = resnet_module(b2, unit=64, stride1=(4,4), stride2=(3,3), stride3=(4,4), mid_activation='relu',bn=False, std=3e-1)
 b3 = resnet_module(b2, unit=128, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid', bn=False, std=1e-2)
 b3 = resnet_module(b3, unit=128, stride1=(6,1), stride2=(3,3), stride3=(1,6),bn=False, std=1.5e-3)
 b3 = resnet_module(b3, unit=128, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid',bn=True, std=3e-1)
 b3 = Dropout(0.5 )(b3)
 b4 = resnet_module(b3, unit=256, stride1=(3,3), stride2=(1,1), stride3=(3,3), mid_activation='sigmoid', bn=False, std=1e-2)
 b4 = resnet_module(b4, unit=256, stride1=(6,1), stride2=(1,1), stride3=(1,6),bn=False, std=1.5e-3)
 b4 = resnet_module(b4, unit=256, stride1=(2,3), stride2=(1,1), stride3=(2,1), mid_activation='relu',bn=True, std=3e-1)
 b5 = resnet_module(b4, unit=256, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid', bn=True, std=1e-2)
 b5 = resnet_module(b5, unit=256, stride1=(3, 3), stride2=(3,3), stride3=(3,3),bn=False, std=1.5e-3)
 b5 = resnet_module(b5, unit=256, stride1=(4,4), stride2=(3,3), stride3=(4,4), mid_activation='relu',bn=True, std=3e-1)
 b5 = Dropout(0.5 )(b5)
 b6 = resnet_module(b5, unit=256, stride1=(3,3), stride2=(7,7), stride3=(2,3), mid_activation='relu', bn=True, std=1e-2)
 b6 = resnet_module(b6, unit=256, stride1=(4,1), stride2=(3,3), stride3=(1,4),bn=False, std=1.5e-3)
 b6 = resnet_module(b6, unit=256, stride1=(2,2), stride2=(1,1), stride3=(3,2), mid_activation='relu',bn=True, std=3e-1)
 p = fc(b6, unit=100, n=3, activation='relu')
 return p
logreg = LogisticRegression()<predict_on_test>
model = models.load_model('/kaggle/input/volcanonn/res32_v2')
 K.set_value(model.optimizer.learning_rate, 0.0015 )
predict_model(traindf, testdf, predictors_var, outcome_var, logreg )<sort_values>
model.optimizer.learning_rate
coef1.sort_values(ascending=False )<find_best_params>
def kfold_validation(model, X, y, X_test, n_rep=3, n_fold=5, batch_size=32, epochs=300, verbose=1):

 history, index = {}, 0
prediction = np.zeros(( X_test.shape[0]))
preds = np.empty(( n_rep * n_fold, X_test.shape[0]))
kf = RepeatedKFold(n_splits=n_fold, n_repeats=n_rep, random_state=288490)
 for train_indices, val_indices in kf.split(X, y):
print(f'{index + 1}th fold, Validation Indices: ', val_indices[:5])
 X_train, X_val = X[train_indices], X[val_indices]
X_rolling_train, X_rolling_val = X_rolling[train_indices], X_rolling[val_indices]
y_train, y_val = y[train_indices], y[val_indices]
history[index] = model.fit(
 x=[X_train, X_rolling_train],
y=y_train,
epochs=epochs,
batch_size=batch_size,
validation_data=([X_val, X_rolling_val], y_val),
verbose=verbose,
callbacks=[
ReduceLROnPlateau(factor=0.96, patience=10, verbose=True, monitor='val_loss', min_lr=1e-45, min_delta=1e-4),
EarlyStopping(
 monitor='val_loss',
patience=25,
restore_best_weights=True,
min_delta=1e4,
verbose=True 
)
 ] 
)
 model_prediction = model.predict([X_test, X_rolling_test],
batch_size=batch_size,
verbose=False ).reshape(X_test.shape[0])
 model_prediction = submit_prediction(model_prediction,
write=True,
file_name=f"sub{index}",
Return=True)
 preds[index] = model_prediction
index += 1
print('
p = pd.DataFrame(preds)
 p = p.sum() /(n_fold * n_rep)
 avg_pred = submit_prediction(p,
write=True,
file_name=f"S_avg",
Return=True)
 return model, preds, avg_pred, history
svc = SVC()<predict_on_test>
model, preds, avg_pred, history = kfold_validation(model, X, y_aug, X_test, 6, 3, 32, epochs=500, verbose=0)
 np.save('preds_V16.npy', preds)
 model.save(f'/kaggle/working/{MODEL_NAME}' )