Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<save_to_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
import os
y_test = np.argmax(y_test, axis=1)
 submission = pd.read_csv(os.path.join(".. /input/",'sample_submission.csv'))
submission['surface'] = le.inverse_transform(y_test)
 submission.to_csv('submission2.csv', index=False)
 submission.head(10)
 create_download_link(filename='submission2.csv' )<predict_on_test>
def k_folds_gbm(X, y, X_test, k,n_est):
folds = StratifiedKFold(n_splits = k, shuffle=True, random_state=2019)
 y_test = np.zeros(( X_test.shape[0], 9))
y_oof = np.zeros(( X.shape[0]))
score = 0
for i,(train_idx, val_idx)in enumerate(folds.split(X, y)) :
clf = XGBClassifier(n_estimators = n_est, n_jobs = -1,learning_rate = 0.2,objective='multi:softmax',eval_metric='merror')
 clf.fit(X_train.iloc[train_idx], y[train_idx])
 y_oof[val_idx] = clf.predict(X.iloc[val_idx])
 y_test += clf.predict_proba(X_test)/ folds.n_splits
score += clf.score(X.iloc[val_idx], y[val_idx])
 print('Fold: {} score: {}'.format(i,clf.score(X.iloc[val_idx], y[val_idx])))
 print('Avg Accuracy', score / folds.n_splits)
 return y_oof, y_test 
<train_model>
class myscaler:
def __init__(self, data):
self.shape = data.shape
if len(data.shape)== 3:
self.diff_stat = []
self.scaler(data)
 self.transform = self._transform
elif len(data.shape)== 2:
scaler = StandardScaler()
scaler.fit(data)
 self.transform = scaler.transform
elif len(data.shape == 1):
print("dim error")
 self.transform = None
def scaler(self, data):
features = data.shape[-1]
for i in range(features):
self.diff_stat.append([np.mean(data[...,i]), np.std(data[...,i])])
 def _transform(self, data):
assert data.shape[-1] == self.shape[-1]
features = data.shape[-1]
for i in range(features):
data[...,i] =(data[...,i] - self.diff_stat[i][0])/self.diff_stat[i][1]
return data
def transform(self,data):
return self.transform(data)
 def gen_feature(data):
features = []
features.append(data.abs().min())
 features.append(data.abs().max())
 features.append(data.abs().mean())
 features.append(data.min())
 features.append(data.max())
 features.append(data.mean())
 features.append(data.kurtosis())
 features.append(data.skew())
 for i in np.arange(0.01, 0.1, 0.01):
features.append(np.quantile(data, i))
for i in np.arange(0.9, 1, 0.01):
features.append(np.quantile(data, i))
data = data.diff().dropna()
features.append(data.abs().min())
 features.append(data.abs().max())
 features.append(data.abs().mean())
 features.append(data.min())
 features.append(data.max())
 features.append(data.mean())
 features.append(data.kurtosis())
 features.append(data.skew())
 for i in np.arange(0.01, 0.1, 0.01):
features.append(np.quantile(data, i))
for i in np.arange(0.9, 1, 0.01):
features.append(np.quantile(data, i))
return pd.Series(features)
 def rolling_feature(data, window_size = 2000, step = 2000):
def feature_small_set(data):
features = []
features.append(data.min())
 features.append(data.max())
 features.append(data.mean())
 features.append(data.std())
 data = np.diff(data)
 features.append(data.min())
 features.append(data.max())
 features.append(data.mean())
 features.append(data.std())
 for i in np.arange(0.9, 1, 0.01):
features.append(np.quantile(data, i))
return features
ans = []
for i in range(window_size, len(data), step):
ans.append(feature_small_set(data[(i-window_size):i]))
return np.array(ans)
 def check_quake(data):
return(data.diff(-1 ).dropna() >= 0 ).all()
def load_batch(data,labels, batch_size=20, shuffle=True):
assert len(data)== len(labels)
 size = len(data)
 for i in range(0, size, batch_size):
if(i+batch_size > size):
yield data[i:size], labels[i:size]
else:
yield data[i:i+batch_size], labels[i:i + batch_size]
y_oof, y_test = k_folds_gbm(X_train, y_raw['surface'], X_test, k= 50,n_est=500)
<save_to_csv>
features_name = ["abs_min", "abs_max", "abs_mean", "min", 'max', 'mean', 'kurtosis', 'skew'] + ["quantile" + f"{i:.3}" for i in np.arange(0.01, 0.1, 0.01)] + ["quantile" + f"{i:.3}" for i in np.arange(0.9, 1, 0.01)]
features_name = features_name + ["diff_" + i for i in features_name ]
y_test = np.argmax(y_test, axis=1)
 submission = pd.read_csv(os.path.join(".. /input/",'sample_submission.csv'))
submission['surface'] = le.inverse_transform(y_test)
 submission.to_csv('submission3.csv', index=False)
 submission.head(10)
 create_download_link(filename='submission3.csv' )<set_options>
def gen_data(feature_fun = gen_feature, test_size=100, size = None, shuffle=True, start_from = None):
train_gen = pd.read_csv(".. /input/train.csv", chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
 X_train = []
y_train = []
X_test = []
y_test = []
if start_from is None:
start_from = 0
for i, data in enumerate(train_gen):
data.rename({'acoustic_data': "signal", 'time_to_failure': "quaketime"}, inplace=True, axis="columns")
 if i < start_from:
del data
continue
if size is not None and i >= start_from + size:
break
if check_quake(data["quaketime"])== False:
print("quake at", i, "excluded")
 try:
print(X_train[-1].shape)
 except IndexError:
pass
X_train.append(feature_fun(data.signal))
y_train.append(data.quaketime.values[-1])
 del data
gc.collect()
X_train = np.array(X_train)
 y_train = np.array(y_train)
 if shuffle:
index = np.arange(0, len(X_train))
np.random.shuffle(index)
 X_train = X_train[index]
y_train = y_train[index]
X_test = X_train[:test_size]
y_test = y_train[:test_size]
X_train = X_train[test_size:]
y_train = y_train[test_size:]
print(X_train.shape, y_train.shape)
 print(X_test.shape, y_test.shape)
 return X_train, y_train, X_test, y_test
warnings.filterwarnings('ignore')
 logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)
 pd.set_option("max_columns", 200)
 pd.set_option("max_rows", 200)
 gc.enable()<load_from_csv>
X_train, y_train, X_test, y_test = gen_data(size=2000,test_size=200 )
x_train = pd.read_csv('.. /input/X_train.csv')
 y_train = pd.read_csv('.. /input/y_train.csv')
 x_test = pd.read_csv('.. /input/X_test.csv' )<feature_engineering>
scaler1 = StandardScaler()
scaler1.fit(X_train)
 X_train_scaled = scaler1.transform(X_train)
 X_test = scaler1.transform(X_test )
tsfresh_train = extract_features(x_train.drop('row_id', axis=1), column_id='series_id',
column_sort='measurement_number')
 impute(tsfresh_train);<feature_engineering>
train_pool = Pool(X_train_scaled, y_train)
 cat_models = CatBoostRegressor(iterations=10000, loss_function="MAE", boosting_type="Ordered")
 cat_models.fit(X_train_scaled, y_train, silent=True)
 print(cat_models.best_score_)
relevant_train_features = set()
for label in y_train['surface'].unique() :
y_train_binary =(y_train['surface'].values == label ).astype(int)
 print('=='*20); print(y_train_binary); print('=='*20);
X_train_filtered = select_features(tsfresh_train, y_train_binary, fdr_level=0.382)
 print('=='*20);
print("Number of relevant features for class {}: {}/{}".format(
 label, X_train_filtered.shape[1], tsfresh_train.shape[1]))
print('=='*20);
relevant_train_features = relevant_train_features.union(set(X_train_filtered.columns))<feature_engineering>
test_pool = Pool(X_test, y_test)
 pre = cat_models.predict(X_test)
 print("catboost score:", np.mean(np.abs(y_test - pre)) )
tsfresh_test = extract_features(x_test.drop('row_id', axis=1), column_id='series_id',
column_sort='measurement_number')
 impute(tsfresh_test); 
<drop_column>
rf = RandomForestRegressor(max_depth=2, n_estimators=3000,max_features=40, criterion="mae")
 rf.fit(X_train_scaled, y_train)
tsfresh_train = tsfresh_train[list(relevant_train_features)]
tsfresh_test = tsfresh_test[list(relevant_train_features)]<features_selection>
print("rf score:", rf.score(X_test, y_test.flatten()))
fac_surfaces, surfaces = y_train['surface'].factorize()
train_data = lgb.Dataset(tsfresh_train, label=fac_surfaces)
 params={'learning_rate': 0.1, 'objective':'multiclass', 'metric':'multi_error',
'num_class':9, 'verbose': 1, 'random_state':311,
'bagging_fraction': 0.7, 'feature_fraction': 1.0}
num_round = 15000
light = lgb.train(params, train_data, num_round)
 pred = light.predict(tsfresh_test)
 feature_importances = light.feature_importance()
feature_names = tsfresh_test.columns.values<categorify>
print("svm score:", np.mean(np.abs(y_test - svm.predict(X_test))))
pred[:,0].shape
fac_surfaces, surfaces = y_train['surface'].factorize()
final_pred = pd.Series(np.argmax(pred, axis=1))
surface_dict = {}
for n, s in enumerate(surfaces):
surface_dict[n] = s
final_pred = final_pred.map(surface_dict )<save_to_csv>
params = {'num_leaves': 128,
'min_child_samples': 79,
'objective': 'gamma',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"subsample_freq": 5,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501,
'colsample_bytree': 0.1
}
lgbm = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)
 lgbm.fit(X_train_scaled, y_train, eval_set=[(X_train_scaled, y_train),(X_test, y_test)], eval_metric='mae',
verbose=10000)
submission = pd.DataFrame({
"series_id": list(range(3816)) ,
"surface": final_pred
})
 submission.to_csv('submission.csv', index=False )<set_options>
print("lgbm score:", np.mean(np.abs(y_test - lgbm.predict(X_test))))
%load_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
def predict(models, x, weights = None):
ans = []
y = 0
for i, model in enumerate(models):
if weights is None:
y += model.predict(x)*(1/len(models))
else:
y += model.predict(x)*weights[i]
return y
models = [cat_models, svm, rf, lgbm]
from fastai.imports import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics
from fastai.tabular import *
from fastai.collab import *
from IPython.display import HTML
from sklearn.preprocessing import LabelEncoder<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 index = list(submission.index )
df_raw = pd.read_csv(f'{PATH}/X_train.csv', low_memory=False)
 y_raw = pd.read_csv(f'{PATH}/y_train.csv', low_memory=False)
 df_test = pd.read_csv(f'{PATH}/X_test.csv', low_memory=False)
 sub = pd.read_csv(f'{PATH}/sample_submission.csv' )<filter>
test = []
for seg_id in index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 test.append(gen_feature(seg['acoustic_data']))
test = np.array(test)
 test = scaler1.transform(test )
df_raw[df_raw.isnull().any(axis=1)]<filter>
y_pre1 = predict(models, test, [0.3,0.3,0.2,0.2] )
y_raw[y_raw.isnull().any(axis=1)]<filter>
try:
del X_train, y_train, X_test, y_test
except:
pass
X_train, y_train, X_test, y_test = gen_data(feature_fun = rolling_feature,size=2000,test_size=200 )
df_test[df_test.isnull().any(axis=1)]<categorify>
class MyLSTM:
def __init__(self, data, labels, valid_data, valid_labels,input_shape = 149):
self.nn = self.bulid_lstm(data.shape[1], data.shape[2])
 cb = EarlyStopping(monitor='val_loss', patience=1, verbose=0, mode='min')
 self.nn.compile(optimizer="adam", loss="MAE")
 self.input_shape =(data.shape[1], data.shape[2])
 self.data = data
self.labels = labels
self.valid =(valid_data,valid_labels)
 def bulid_lstm(self, input_shape, feature_shape):
seq = keras.Input(shape=(input_shape, feature_shape))
x = CuDNNGRU(48 )(seq)
 x = Dense(10, activation='relu' )(x)
 x = Dense(units=1 )(x)
 model = Model(inputs=seq, outputs=x)
 model.summary()
return model
def train_on_batch(self, epochs = 50, batch_size=20, intervals=10):
losses = []
test_losses = []
for epoch in range(epochs):
for batch_i, data in enumerate(load_batch(self.data, self.labels, batch_size=batch_size)) :
loss = self.nn.train_on_batch(data[0], data[1],)
 losses.append(loss)
 valid = np.mean(np.abs(self.valid[1] - self.nn.predict(self.valid[0])))
 test_losses.append(np.mean(valid))
if batch_i%intervals == 0 or(batch_i == batch_size-1):
print(f"[Epoch {epoch}/{epochs}] [Batch {batch_i}] [MAE loss {loss}] [valid loss {valid}]")
 plt.plot(losses, label="training loss")
 plt.plot(test_losses, label="validation loss")
 plt.legend()
plt.show()
if epoch > 4:
stop =(test_losses[-1] > np.min(test_losses)+ 0.1)
 if stop:
print(f"stop train at [Epoch {epoch}/{epochs}]")
 break
def predict(self, test_data):
return self.nn.predict(test_data)
le = LabelEncoder()
le.fit(y_raw['surface'])
 y_raw['surface'] = le.transform(y_raw['surface'] )<prepare_x_and_y>
lstm = MyLSTM(X_train, y_train, X_test, y_test )
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def fe(df):
df['total_angular_velocity'] =(df['angular_velocity_X'] ** 2 + df['angular_velocity_Y'] ** 2 + df['angular_velocity_Z'] ** 2)** 0.5
df['total_linear_acceleration'] =(df['linear_acceleration_X'] ** 2 + df['linear_acceleration_Y'] ** 2 + df['linear_acceleration_Z'] ** 2)** 0.5
df['total_xyz'] =(df['orientation_X']**2 + df['orientation_Y']**2 +df['orientation_Z'])**0.5
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
temp_df = pd.DataFrame()
for col in df.columns[3:]:
temp_df[col + '_mean'] = df.groupby(['series_id'])[col].mean()
temp_df[col + '_median'] = df.groupby(['series_id'])[col].median()
temp_df[col + '_max'] = df.groupby(['series_id'])[col].max()
temp_df[col + '_min'] = df.groupby(['series_id'])[col].min()
temp_df[col + '_std'] = df.groupby(['series_id'])[col].std()
temp_df[col + '_range'] = temp_df[col + '_max'] - temp_df[col + '_min']
temp_df[col + '_maxtoMin'] = temp_df[col + '_max'] / temp_df[col + '_min']
temp_df[col + '_mean_abs_chg'] = df.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
temp_df[col + '_abs_min'] = df.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 temp_df[col + '_abs_max'] = df.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 temp_df[col + '_abs_avg'] =(temp_df[col + '_abs_min'] + temp_df[col + '_abs_max'])/2
return temp_df<feature_engineering>
lstm.train_on_batch(batch_size=50,epochs=50 )
%%time
df_train = fe(df_raw)
 df_test = fe(df_test )<feature_engineering>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 index = list(submission.index )
df_train['surface'] = y_raw['surface']<load_from_csv>
test = []
for seg_id in index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 test.append(rolling_feature(seg['acoustic_data']))
test = np.array(test)
 test = scaler.transform(test)
def create_download_link(title = "Download CSV file", filename = "data.csv"):
html = '<a href={filename}>{title}</a>'
html = html.format(title=title,filename=filename)
 return HTML(html )<import_modules>
y_pre2 = lstm.predict(test )
<prepare_x_and_y>
y_pre = 0.6*y_pre1 + 0.4*y_pre
df, y = df_train.drop('surface',axis=1), df_train.surface<drop_column>
submission.time_to_failure = y_pre
submission.to_csv('submission.csv' )