Kaggle competition: LANL Earthquake Prediction
predict = xgb.predict(X_test )<compute_test_metric>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm
from tqdm import tqdm_notebook
from scipy.signal import hilbert
from scipy.signal import hann
from scipy.signal import convolve
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict)))) 
<train_model>
train_data = pd.read_csv('.. /input/train.csv', dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float32} )
xgb.fit(df_train, df_train_target )<predict_on_test>
seg_length = 150000
total_samples = int(np.floor(( train_data.shape[0])/ seg_length))
x_train = pd.DataFrame(index = range(total_samples), dtype = np.float64)
 y_train = pd.DataFrame(index = range(total_samples), columns = ['time_to_failure'], dtype = np.float64 )
predict2 = xgb.predict(df_test)
 predict_y =(predict2 )<predict_on_test>
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
 for value in tqdm_notebook(range(total_samples)) :
sample = train_data.iloc[value*seg_length : value*seg_length + seg_length]
x = pd.Series(sample['acoustic_data'].values)
 y = sample['time_to_failure'].values[-1]
y_train.loc[value, 'time_to_failure'] = y
x_train.loc[value, 'AVERAGE'] = x.mean()
x_train.loc[value, 'STD'] = x.std()
x_train.loc[value, 'MAX'] = x.max()
x_train.loc[value, 'MIN'] = x.min()
x_train.loc[value, 'SUM'] = x.sum()
x_train.loc[value, 'MEAN_CHANGE_ABS'] = np.mean(np.diff(x))
x_train.loc[value, 'MEAN_CHANGE_RATE'] = calc_change_rate(x)
 x_train.loc[value, 'MAX_TO_MIN'] = x.max() / np.abs(x.min())
 x_train.loc[value, 'MAX_TO_MIN_DIFF'] = x.max() - np.abs(x.min())
 x_train.loc[value, 'COUNT_BIG'] = len(x[np.abs(x)> 500])
 x_train.loc[value, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
x_train.loc[value, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
x_train.loc[value, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
x_train.loc[value, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
x_train.loc[value, 'STD_FIRST_10000'] = x[:10000].std()
x_train.loc[value, 'STD_LAST_10000'] = x[-10000:].std()
x_train.loc[value, 'STD_FIRST_50000'] = x[:50000].std()
x_train.loc[value, 'STD_LAST_50000'] = x[-50000:].std()
x_train.loc[value, 'ABS_AVERAGE'] = np.abs(x ).mean()
x_train.loc[value, 'ABS_STD'] = np.abs(x ).std()
x_train.loc[value, 'ABS_MAX'] = np.abs(x ).max()
x_train.loc[value, 'ABS_MIN'] = np.abs(x ).min()
x_train.loc[value, '10Q'] = np.percentile(x, 0.10)
 x_train.loc[value, '25Q'] = np.percentile(x, 0.25)
 x_train.loc[value, '50Q'] = np.percentile(x, 0.50)
 x_train.loc[value, '75Q'] = np.percentile(x, 0.75)
 x_train.loc[value, '90Q'] = np.percentile(x, 0.90)
 x_train.loc[value, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
x_train.loc[value, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
x_train.loc[value, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
x_train.loc[value, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
x_train.loc[value, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
x_train.loc[value, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
x_train.loc[value, 'KURTOSIS'] = x.kurtosis()
x_train.loc[value, 'SKEW'] = x.skew()
x_train.loc[value, 'MEDIAN'] = x.median()
x_train.loc[value, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
x_train.loc[value, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()
for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
x_train.loc[value, 'AVG_ROLL_STD' + str(windows)] = x_roll_std.mean()
x_train.loc[value, 'STD_ROLL_STD' + str(windows)] = x_roll_std.std()
x_train.loc[value, 'MAX_ROLL_STD' + str(windows)] = x_roll_std.max()
x_train.loc[value, 'MIN_ROLL_STD' + str(windows)] = x_roll_std.min()
x_train.loc[value, '1Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.01)
 x_train.loc[value, '5Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.05)
 x_train.loc[value, '95Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.95)
 x_train.loc[value, '99Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.99)
 x_train.loc[value, 'AV_CHANGE_ABS_ROLL_STD' + str(windows)] = np.mean(np.diff(x_roll_std))
x_train.loc[value, 'ABS_MAX_ROLL_STD' + str(windows)] = np.abs(x_roll_std ).max()
x_train.loc[value, 'AVG_ROLL_MEAN' + str(windows)] = x_roll_mean.mean()
x_train.loc[value, 'STD_ROLL_MEAN' + str(windows)] = x_roll_mean.std()
x_train.loc[value, 'MAX_ROLL_MEAN' + str(windows)] = x_roll_mean.max()
x_train.loc[value, 'MIN_ROLL_MEAN' + str(windows)] = x_roll_mean.min()
x_train.loc[value, '1Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 x_train.loc[value, '5Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 x_train.loc[value, '95Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 x_train.loc[value, '99Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 x_train.loc[value, 'AV_CHANGE_ABS_ROLL_MEAN' + str(windows)] = np.mean(np.diff(x_roll_mean))
x_train.loc[value, 'ABS_MAX_ROLL_MEAN' + str(windows)] = np.abs(x_roll_mean ).max()
lr = LinearRegression()
lr.fit(X_train,y_train)
 y_pred = lr.predict(X_test)
<create_dataframe>
y_train.isnull().sum()
coeff_df = pd.DataFrame(lr.coef_,df_train.columns,columns=['Coefficient'])
 coeff_df<compute_test_metric>
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error
lr.score(X_test,y_test)
<compute_test_metric>
scaler = StandardScaler()
scaler.fit(x_train)
 x_train_scaled = scaler.transform(x_train)
 y_train_flatten = y_train.values.ravel()
print('RMSE: {}'.format(np.sqrt(metrics.mean_squared_error(y_test, y_pred)))) 
<save_to_csv>
x_train_dataframe = pd.DataFrame(x_train_scaled)
 x_train_dataframe.head(10 )
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y
})
 submission.to_csv('submission.csv', index=False )<import_modules>
x_train1, x_test1, y_train1, y_test1 = train_test_split(x_train, y_train, test_size = 0.2, random_state = 101 )
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import skew
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from itertools import product
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import mean_squared_error<load_from_csv>
scaler = StandardScaler()
scaler.fit(x_train1)
 x_train_scaled1 = scaler.transform(x_train1)
 x_test_scaled1 = scaler.transform(x_test1)
 y_train_flatten1 = y_train1.values.ravel()
y_test_flatten1 = y_test1.values.ravel()
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
import keras
from keras.layers import Dense
from keras.layers import BatchNormalization
from keras.layers import Activation
from keras.layers import Dropout
from keras.models import Sequential
from keras import optimizers
features = ['GrLivArea','OverallQual','2ndFlrSF','TotalBsmtSF','1stFlrSF','GarageCars','YearBuilt', 'GarageArea','ExterQual',
'YearRemodAdd','FireplaceQu','GarageYrBlt','FullBath','MSSubClass','LotArea','Fireplaces','KitchenQual','MSZoning',
'GarageType','BsmtFinSF1','Neighborhood','BsmtQual','TotRmsAbvGrd','HalfBath','BldgType','GarageFinish',
'Foundation','BedroomAbvGr','HouseStyle','CentralAir','OpenPorchSF','HeatingQC','BsmtFinType1','BsmtUnfSF',
'GarageCond','GarageQual','KitchenAbvGr','OverallCond','BsmtCond','MasVnrArea','BsmtFullBath','Exterior1st',
'Exterior2nd','PavedDrive','WoodDeckSF','LandContour','MasVnrType','BsmtFinType2','Functional','Fence']<concatenate>
kernel_init = 'normal'
def Model(input_dim, activation, classes):
model = Sequential()
model.add(Dense(1024, kernel_initializer = kernel_init, input_dim = input_dim))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(512, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(64, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(classes, kernel_initializer = kernel_init))
model.add(Activation('linear'))
return model
train = train[features+['SalePrice']]
test = test[['Id']+features]
all_data = pd.concat(( train.loc[:,'GrLivArea':'Fence'],
test.loc[:,'GrLivArea':'Fence']))<feature_engineering>
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
model = Model(input_dim = input_dim, activation = activation, classes = classes)
train['SalePrice'] = np.log1p(train['SalePrice'])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<data_type_conversions>
optim = optimizers.Adam(lr = 0.001, decay = 0.001 / 32)
 model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'] )
for feature in all_data.select_dtypes(exclude=['object'] ).columns:
all_data[feature].fillna(all_data[feature].median() , inplace = True)
 for feature in all_data.select_dtypes(include=['object'] ).columns:
all_data[feature].fillna(all_data[feature].value_counts().idxmax() , inplace = True )<feature_engineering>
h = model.fit(x_train_scaled1, y_train_flatten1, epochs = 100, batch_size = 32, verbose = 1 )
all_data['TotalSF'] =(all_data['TotalBsmtSF']
+ all_data['1stFlrSF']
+ all_data['2ndFlrSF'])
 all_data['YrBltAndRemod'] = all_data['YearBuilt'] + all_data['YearRemodAdd']
all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1']
+ all_data['1stFlrSF']
+ all_data['2ndFlrSF'] 
)
 all_data['Total_Bathrooms'] =(all_data['FullBath']
+(0.5 * all_data['HalfBath'])
 + all_data['BsmtFullBath'] 
)
 all_data['Total_porch_sf'] =(all_data['OpenPorchSF']
+ all_data['WoodDeckSF']
)<feature_engineering>
prediction_score = []
mae_scores = []
predictions = model.predict(x_test_scaled1, verbose = 1, batch_size = 32)
 prediction_score.append(predictions)
 mae = mean_absolute_error(predictions, y_test_flatten1)
 mae_scores.append(mae )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<categorify>
print(np.mean(mae_scores))
print(all_data.shape )<prepare_x_and_y>
predictions = np.hstack([p.reshape(-1,1)for p in prediction_score])
 predictions = np.mean(predictions, axis = 1)
 print(mean_absolute_error(predictions, y_test1))
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice<count_values>
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
 submission = pd.read_csv('.. /input/sample_submission.csv', index_col = 'seg_id')
 X_test = pd.DataFrame(columns = x_train.columns, dtype = np.float64, index = submission.index)
 for id in tqdm_notebook(X_test.index):
seg = pd.read_csv('.. /input/test/' + id + '.csv')
 x = pd.Series(seg['acoustic_data'].values)
 X_test.loc[id, 'AVERAGE'] = x.mean()
X_test.loc[id, 'STD'] = x.std()
X_test.loc[id, 'MAX'] = x.max()
X_test.loc[id, 'MIN'] = x.min()
X_test.loc[id, 'SUM'] = x.sum()
X_test.loc[id, 'MEAN_CHANGE_ABS'] = np.mean(np.diff(x))
X_test.loc[id, 'MEAN_CHANGE_RATE'] = calc_change_rate(x)
 X_test.loc[id, 'MAX_TO_MIN'] = x.max() / np.abs(x.min())
 X_test.loc[id, 'MAX_TO_MIN_DIFF'] = x.max() - np.abs(x.min())
 X_test.loc[id, 'COUNT_BIG'] = len(x[np.abs(x)> 500])
 X_test.loc[id, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
X_test.loc[id, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
X_test.loc[id, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
X_test.loc[id, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
X_test.loc[id, 'STD_FIRST_10000'] = x[:10000].std()
X_test.loc[id, 'STD_LAST_10000'] = x[-10000:].std()
X_test.loc[id, 'STD_FIRST_50000'] = x[:50000].std()
X_test.loc[id, 'STD_LAST_50000'] = x[-50000:].std()
X_test.loc[id, 'ABS_AVERAGE'] = np.abs(x ).mean()
X_test.loc[id, 'ABS_STD'] = np.abs(x ).std()
X_test.loc[id, 'ABS_MAX'] = np.abs(x ).max()
X_test.loc[id, 'ABS_MIN'] = np.abs(x ).min()
X_test.loc[id, '10Q'] = np.percentile(x, 0.10)
 X_test.loc[id, '25Q'] = np.percentile(x, 0.25)
 X_test.loc[id, '50Q'] = np.percentile(x, 0.50)
 X_test.loc[id, '75Q'] = np.percentile(x, 0.75)
 X_test.loc[id, '90Q'] = np.percentile(x, 0.90)
 X_test.loc[id, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
X_test.loc[id, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
X_test.loc[id, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
X_test.loc[id, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
X_test.loc[id, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
X_test.loc[id, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
X_test.loc[id, 'KURTOSIS'] = x.kurtosis()
X_test.loc[id, 'SKEW'] = x.skew()
X_test.loc[id, 'MEDIAN'] = x.median()
X_test.loc[id, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
X_test.loc[id, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()
for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_test.loc[id, 'AVG_ROLL_STD' + str(windows)] = x_roll_std.mean()
X_test.loc[id, 'STD_ROLL_STD' + str(windows)] = x_roll_std.std()
X_test.loc[id, 'MAX_ROLL_STD' + str(windows)] = x_roll_std.max()
X_test.loc[id, 'MIN_ROLL_STD' + str(windows)] = x_roll_std.min()
X_test.loc[id, '1Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_test.loc[id, '5Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_test.loc[id, '95Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_test.loc[id, '99Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_test.loc[id, 'AV_CHANGE_ABS_ROLL_STD' + str(windows)] = np.mean(np.diff(x_roll_std))
X_test.loc[id, 'ABS_MAX_ROLL_STD' + str(windows)] = np.abs(x_roll_std ).max()
X_test.loc[id, 'AVG_ROLL_MEAN' + str(windows)] = x_roll_mean.mean()
X_test.loc[id, 'STD_ROLL_MEAN' + str(windows)] = x_roll_mean.std()
X_test.loc[id, 'MAX_ROLL_MEAN' + str(windows)] = x_roll_mean.max()
X_test.loc[id, 'MIN_ROLL_MEAN' + str(windows)] = x_roll_mean.min()
X_test.loc[id, '1Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_test.loc[id, '5Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_test.loc[id, '95Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_test.loc[id, '99Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_test.loc[id, 'AV_CHANGE_ABS_ROLL_MEAN' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_test.loc[id, 'ABS_MAX_ROLL_MEAN' + str(windows)] = np.abs(x_roll_mean ).max()
categoricals = train.select_dtypes(include=['object'] ).columns
for c in categoricals:
for cell in X_train[c].unique().tolist() :
num = X_train[c].value_counts() [cell]
X_train[c] = X_train[c].replace({cell: num} )<filter>
X_test.isnull().sum()
X_train[categoricals]<categorify>
X_test_scaled = scaler.transform(X_test)
 print(X_test_scaled.shape )
X_train[X_train.select_dtypes(exclude=['object'] ).columns]<normalization>
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
history = dict()
prediction_scores = dict()
for i in np.arange(0, 15):
optim = optimizers.Adam(lr = 0.001, decay = 0.001 / 32)
 ensemble_model = Model(input_dim = input_dim, activation = activation, classes = classes)
 ensemble_model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'])
 print('TRAINING MODEL NO : {}'.format(i))
H = ensemble_model.fit(x_train_scaled, y_train_flatten,
batch_size = 32,
epochs = 200,
verbose = 1)
 history[i] = H
predictions = ensemble_model.predict(X_test_scaled, verbose = 1, batch_size = 32)
 prediction_scores[i] = predictions
<compute_train_metric>
prediction1 = np.hstack([p.reshape(-1,1)for p in prediction_scores.values() ])
 prediction1 = np.mean(prediction1, axis = 1)
 print(prediction1.shape )
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )<define_search_space>
submission['time_to_failure'] = prediction1
submission.to_csv('submission.csv' )