Kaggle competition: INGV - Volcanic Eruption Prediction
dataset.isnull().sum()<categorify>
X = pd.read_csv(".. /input/preprocessed-volcano-data/X.csv",sep=";",index_col=0)
 Y = pd.read_csv(".. /input/preprocessed-volcano-data/Y.csv",sep=";",index_col=0 )
embarked_dummy = pd.get_dummies(dataset['Embarked'] )<merge>
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X,Y["time_to_eruption"],test_size=500,random_state=1234)
 reg = LinearRegression()
cols = [col for col in X.columns if "max" in col and not "argmax" in col]
reg.fit(Xtrain[cols].values, Ytrain)
 pred = reg.predict(Xtest[cols].values)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(Ytest,pred)[0,1]**2
print(R2 )
dataset = dataset.join(embarked_dummy )<data_type_conversions>
gridsearch = GridSearchCV(estimator=RandomForestRegressor() ,
param_grid={"max_features" : [50, 100, 150],
"min_samples_leaf" : [1, 5],
"n_estimators" : [1000]},
cv=5,n_jobs=5,scoring="neg_mean_absolute_error",verbose=10)
 gridsearch.fit(Xtrain, Ytrain)
 res_RF = pd.DataFrame({"max_features" : [list(x.values())[0] for x in gridsearch.cv_results_["params"]],
"min_samples_leaf" : [list(x.values())[1] for x in gridsearch.cv_results_["params"]],
"mean_mae" : np.round(gridsearch.cv_results_["mean_test_score"],4),
"sd_mae" : np.round(gridsearch.cv_results_["std_test_score"],4)})
 print(res_RF )
dataset['Fare'].fillna(dataset['Fare'].median() , inplace=True )<categorify>
gridsearch = GridSearchCV(estimator=ExtraTreesRegressor() ,
param_grid={"max_features" : [50, 100, 150],
"min_samples_leaf" : [1, 5],
"n_estimators" : [1000]},
cv=5,n_jobs=5,scoring="neg_mean_absolute_error",verbose=10)
 gridsearch.fit(Xtrain, Ytrain)
 res_ET = pd.DataFrame({"max_features" : [list(x.values())[0] for x in gridsearch.cv_results_["params"]],
"min_samples_leaf" : [list(x.values())[1] for x in gridsearch.cv_results_["params"]],
"mean_mae" : np.round(gridsearch.cv_results_["mean_test_score"],4),
"sd_mae" : np.round(gridsearch.cv_results_["std_test_score"],4)})
 print(res_ET )
le = LabelEncoder()<feature_engineering>
ET = ExtraTreesRegressor(max_features=164,min_samples_leaf=1)
 ET.fit(Xtrain, Ytrain)
 pred = ET.predict(Xtest)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(Ytest,pred)[0,1]**2
print(R2 )
dataset['FareBand'] = pd.qcut(dataset['Fare'], 4 )<categorify>
def FeatureSelection(X,Y,nfolds=5,random_state=123):
XX = X.values
n_iter = 30
cvmeanMAE = np.zeros(n_iter)
 cvminMAE = np.zeros(n_iter)
 cvmaxMAE = np.zeros(n_iter)
 cols = X.columns
remaining_cols = []
cvsplit = KFold(n_splits=nfolds, shuffle=True, random_state=random_state)
 for i in range(n_iter):
print("Variable selection step " + str(i + 1)+ " - " + str(XX.shape[1])+ " features remaining")
 rf = ExtraTreesRegressor(max_features=XX.shape[1], min_samples_leaf=1, n_estimators=1000)
 cv = cross_validate(estimator=rf, X=XX, y=Y, cv=cvsplit, n_jobs=5,
scoring="neg_mean_absolute_error")
 cvmeanMAE[i] = np.mean(-cv["test_score"])
 cvminMAE[i] = np.min(-cv["test_score"])
 cvmaxMAE[i] = np.max(-cv["test_score"])
 rf.fit(XX, Y)
 varimp = rf.feature_importances_
ind = np.argsort(varimp)[5:]
XX = XX[:, ind]
cols = cols[ind]
remaining_cols.append(cols)
 res = {"remaining_cols" : remaining_cols,
"cvmeanMAE" : cvmeanMAE,
"cvminMAE" : cvminMAE,
"cvmaxMAE" : cvmaxMAE}
return res
tmp = FeatureSelection(Xtrain, Ytrain)
 pyplot.bar(np.arange(len(tmp["cvmeanMAE"])) ,height=tmp["cvmeanMAE"],
yerr=np.concatenate([(tmp["cvmeanMAE"]-tmp["cvminMAE"] ).reshape(-1,1), 
(tmp["cvmaxMAE"]-tmp["cvmeanMAE"] ).reshape(-1,1)],axis=1 ).T )
dataset['FareClass'] = le.fit_transform(dataset['FareBand'] )<data_type_conversions>
sel_cols = ['S1_max', 'S10_Q75', 'S3_Q50', 'S1_sumNA', 'S3_Q1', 'S3_max', 'S9_std',
'S6_Q5', 'S7_Q1', 'S10_Q95', 'S5_Q90', 'S2_Q5', 'S5_Q5', 'S6_Q10',
'S7_Q50', 'S1_Q95', 'S3_Q75', 'S5_Q10', 'S8_Q99', 'S4_Q95', 'S3_sumNA',
'S7_Q90', 'S8_Q50', 'S6_Q90', 'S10_Q90', 'S2_sumNA', 'S4_Q99', 'S2_Q75',
'S9_Q90', 'S9_Q25', 'S1_Q99', 'S6_Q1', 'S1_Q5', 'S7_Q99', 'S3_Q5',
'S3_Q95', 'S5_Q99', 'S4_Q1', 'S9_Q50', 'S2_std', 'S1_Q75', 'S2_Q50',
'S7_Q5', 'S10_Q25', 'S5_sumNA', 'S8_Q5', 'S10_Q1']
<categorify>
Xtrain = Xtrain[sel_cols]
Xtest = Xtest[sel_cols]
ET = ExtraTreesRegressor(max_features=47,min_samples_leaf=1,n_estimators=1000)
 ET.fit(Xtrain, Ytrain)
 pred = ET.predict(Xtest)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(pred,Ytest)[0,1]**2
print(R2 )
dataset['Title'] = le.fit_transform(dataset['Title'] )<feature_engineering>
class Network:
def __init__(self,file,nval=500,nepochs=300,batch_size=64):
self.nval = nval
self.nepochs = nepochs
self.batch_size = batch_size
self.file = file
def __build_model(self,input_shape):
net = keras.models.Sequential([
keras.layers.Input(shape=input_shape),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(256, activation="relu"),
keras.layers.Dense(512, activation="relu"),
keras.layers.Dropout(0.5),
keras.layers.Dense(1, activation="linear")
 ])
 return net
def fit(self,X,Y):
Xtrain, Xval, Ytrain, Yval = train_test_split(X, Y, test_size=self.nval)
 scaler = StandardScaler()
scaler.fit(Xtrain)
 Xtrain_sc = scaler.transform(Xtrain)
 Xval_sc = scaler.transform(Xval)
 my = np.mean(Ytrain)
 sy = np.std(Ytrain)
 Ytrain_sc =(Ytrain - my)/ sy
Yval_sc =(Yval - my)/ sy
net = self.__build_model(X.shape[1])
 checkpoint = keras.callbacks.ModelCheckpoint(self.file,monitor="val_loss",
save_best_only=True,mode="min",
verbose=1)
 lr_callback = keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=20)
 net.compile(loss="mean_absolute_error", optimizer=keras.optimizers.Nadam(learning_rate=0.001))
fitted = net.fit(x=Xtrain_sc, y=Ytrain_sc, batch_size=self.batch_size,
epochs=self.nepochs, verbose=1,
callbacks=[lr_callback, checkpoint],
validation_data=(Xval_sc, Yval_sc))
net = keras.models.load_model(self.file)
 fitted = {"net" : net, "scaler" : scaler, "my" : my, "sy" : sy}
self.fitted = fitted
def predict(self,Xtest):
fitted = self.fitted
net = fitted["net"]
scaler = fitted["scaler"]
my = fitted["my"]
sy = fitted["sy"]
pred = net.predict(scaler.transform(Xtest)) [:,0]*sy+my
return pred
dataset['Age*Class'] = dataset['AgeClass'] * dataset['Pclass']<string_transform>
net = Network(file="fitted_network.h5",nval=500,nepochs=300,batch_size=64)
 net.fit(Xtrain,Ytrain)
 pred = net.predict(Xtest)
 mae = np.mean(np.abs(Ytest-pred))
print(mae)
 R2 = np.corrcoef(Ytest,pred)[0,1]**2
print(R2 )
<drop_column><EOS> 
X = pd.read_csv(".. /input/preprocessed-volcano-data/X.csv",sep=";",index_col=0)
 Y = pd.read_csv(".. /input/preprocessed-volcano-data/Y.csv",sep=";",index_col=0)
 Xtest = pd.read_csv(".. /input/preprocessed-volcano-data/Xtest.csv",sep=";",index_col=0)
 segment_id = Xtest.index
X = X[sel_cols]
Xtest = Xtest[sel_cols]
net = Network(file="fitted_network.h5",nval=500,nepochs=300,batch_size=64)
 net.fit(X,Y["time_to_eruption"])
 pred = net.predict(Xtest)
 res = pd.DataFrame({"segment_id" : segment_id, "time_to_eruption" : pred})
 res.to_csv("prediction_test.csv",sep=",",columns=res.columns,index=False )