Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<data_type_conversions>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
def change_datatype_float(df):
float_cols = list(df.select_dtypes(include=['float'] ).columns)
 for col in float_cols:
df[col] = df[col].astype(np.float32)
 change_datatype_float(df_train)
 change_datatype_float(df_test )<count_values>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change )
len(set(df_test.columns)and set(df_train.columns))<data_type_conversions>
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
x = df.acoustic_data.values
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 del df
yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values[-self.chunk_size:]
del df
yield seg_id, x, -999
def get_features(self, x, y, seg_id):

 x = pd.Series(x)
 zc = np.fft.fft(x)
 realFFT = pd.Series(np.real(zc))
imagFFT = pd.Series(np.imag(zc))
main_dict = self.features(x, y, seg_id)
 r_dict = self.features(realFFT, y, seg_id)
 i_dict = self.features(imagFFT, y, seg_id)
 for k, v in r_dict.items() :
if k not in ['target', 'seg_id']:
main_dict[f'fftr_{k}'] = v
for k, v in i_dict.items() :
if k not in ['target', 'seg_id']:
main_dict[f'ffti_{k}'] = v
return main_dict
def features(self, x, y, seg_id):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
percentiles = [1, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 99]
hann_windows = [50, 150, 1500, 15000]
spans = [300, 3000, 30000, 50000]
windows = [10, 50, 100, 500, 1000, 10000]
borders = list(range(-4000, 4001, 1000))
peaks = [10, 20, 50, 100]
coefs = [1, 5, 10, 50, 100]
lags = [10, 100, 1000, 10000]
autocorr_lags = [5, 10, 50, 100, 500, 1000, 5000, 10000]
feature_dict['mean'] = x.mean()
feature_dict['std'] = x.std()
feature_dict['max'] = x.max()
feature_dict['min'] = x.min()
feature_dict['mean_change_abs'] = np.mean(np.diff(x))
feature_dict['abs_max'] = np.abs(x ).max()
feature_dict['abs_mean'] = np.abs(x ).mean()
feature_dict['abs_std'] = np.abs(x ).std()
feature_dict['hmean'] = stats.hmean(np.abs(x[np.nonzero(x)[0]]))
feature_dict['gmean'] = stats.gmean(np.abs(x[np.nonzero(x)[0]]))
for i in range(1, 5):
feature_dict[f'kstat_{i}'] = stats.kstat(x, i)
 feature_dict[f'moment_{i}'] = stats.moment(x, i)
 for i in [1, 2]:
feature_dict[f'kstatvar_{i}'] = stats.kstatvar(x, i)
 for agg_type, slice_length, direction in product(['std', 'min', 'max', 'mean'], [1000, 10000, 50000], ['first', 'last']):
if direction == 'first':
feature_dict[f'{agg_type}_{direction}_{slice_length}'] = x[:slice_length].agg(agg_type)
 elif direction == 'last':
feature_dict[f'{agg_type}_{direction}_{slice_length}'] = x[-slice_length:].agg(agg_type)
 feature_dict['max_to_min'] = x.max() / np.abs(x.min())
 feature_dict['max_to_min_diff'] = x.max() - np.abs(x.min())
 feature_dict['count_big'] = len(x[np.abs(x)> 500])
 feature_dict['sum'] = x.sum()
feature_dict['mean_change_rate'] = calc_change_rate(x)
 for slice_length, direction in product([1000, 10000, 50000], ['first', 'last']):
if direction == 'first':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[:slice_length])
 elif direction == 'last':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[-slice_length:])
 for p in percentiles:
feature_dict[f'percentile_{p}'] = np.percentile(x, p)
 feature_dict[f'abs_percentile_{p}'] = np.percentile(np.abs(x), p)
 feature_dict['trend'] = add_trend_feature(x)
 feature_dict['abs_trend'] = add_trend_feature(x, abs_values=True)
 feature_dict['mad'] = x.mad()
feature_dict['kurt'] = x.kurtosis()
feature_dict['skew'] = x.skew()
feature_dict['med'] = x.median()
feature_dict['Hilbert_mean'] = np.abs(hilbert(x)).mean()
for hw in hann_windows:
feature_dict[f'Hann_window_mean_{hw}'] =(convolve(x, hann(hw), mode='same')/ sum(hann(hw)) ).mean()
feature_dict['classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
feature_dict['classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
feature_dict['classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
feature_dict['classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
feature_dict['classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
feature_dict['classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
feature_dict['classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
feature_dict['classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
ewma = pd.Series.ewm
for s in spans:
feature_dict[f'exp_Moving_average_{s}_mean'] =(ewma(x, span=s ).mean(skipna=True)).mean(skipna=True)
 feature_dict[f'exp_Moving_average_{s}_std'] =(ewma(x, span=s ).mean(skipna=True)).std(skipna=True)
 feature_dict[f'exp_Moving_std_{s}_mean'] =(ewma(x, span=s ).std(skipna=True)).mean(skipna=True)
 feature_dict[f'exp_Moving_std_{s}_std'] =(ewma(x, span=s ).std(skipna=True)).std(skipna=True)
 feature_dict['iqr'] = np.subtract(*np.percentile(x, [75, 25]))
feature_dict['iqr1'] = np.subtract(*np.percentile(x, [95, 5]))
feature_dict['ave10'] = stats.trim_mean(x, 0.1)
 for slice_length, threshold in product([50000, 100000, 150000],
[5, 10, 20, 50, 100]):
feature_dict[f'count_big_{slice_length}_threshold_{threshold}'] =(np.abs(x[-slice_length:])> threshold ).sum()
feature_dict[f'count_big_{slice_length}_less_threshold_{threshold}'] =(np.abs(x[-slice_length:])< threshold ).sum()
feature_dict['range_minf_m4000'] = feature_calculators.range_count(x, -np.inf, -4000)
 feature_dict['range_p4000_pinf'] = feature_calculators.range_count(x, 4000, np.inf)
 for i, j in zip(borders, borders[1:]):
feature_dict[f'range_{i}_{j}'] = feature_calculators.range_count(x, i, j)
 for autocorr_lag in autocorr_lags:
feature_dict[f'autocorrelation_{autocorr_lag}'] = feature_calculators.autocorrelation(x, autocorr_lag)
 feature_dict[f'c3_{autocorr_lag}'] = feature_calculators.c3(x, autocorr_lag)
 for p in percentiles:
feature_dict[f'binned_entropy_{p}'] = feature_calculators.binned_entropy(x, p)
 feature_dict['num_crossing_0'] = feature_calculators.number_crossing_m(x, 0)
 for peak in peaks:
feature_dict[f'num_peaks_{peak}'] = feature_calculators.number_peaks(x, peak)
 for c in coefs:
feature_dict[f'spkt_welch_density_{c}'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': c}])) [0][1]
feature_dict[f'time_rev_asym_stat_{c}'] = feature_calculators.time_reversal_asymmetry_statistic(x, c)
 for w in windows:
x_roll_std = x.rolling(w ).std().dropna().values
x_roll_mean = x.rolling(w ).mean().dropna().values
feature_dict[f'ave_roll_std_{w}'] = x_roll_std.mean()
feature_dict[f'std_roll_std_{w}'] = x_roll_std.std()
feature_dict[f'max_roll_std_{w}'] = x_roll_std.max()
feature_dict[f'min_roll_std_{w}'] = x_roll_std.min()
for p in percentiles:
feature_dict[f'percentile_roll_std_{p}_window_{w}'] = np.percentile(x_roll_std, p)
 feature_dict[f'av_change_abs_roll_std_{w}'] = np.mean(np.diff(x_roll_std))
feature_dict[f'av_change_rate_roll_std_{w}'] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 feature_dict[f'abs_max_roll_std_{w}'] = np.abs(x_roll_std ).max()
feature_dict[f'ave_roll_mean_{w}'] = x_roll_mean.mean()
feature_dict[f'std_roll_mean_{w}'] = x_roll_mean.std()
feature_dict[f'max_roll_mean_{w}'] = x_roll_mean.max()
feature_dict[f'min_roll_mean_{w}'] = x_roll_mean.min()
for p in percentiles:
feature_dict[f'percentile_roll_mean_{p}_window_{w}'] = np.percentile(x_roll_mean, p)
 feature_dict[f'av_change_abs_roll_mean_{w}'] = np.mean(np.diff(x_roll_mean))
feature_dict[f'av_change_rate_roll_mean_{w}'] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 feature_dict[f'abs_max_roll_mean_{w}'] = np.abs(x_roll_mean ).max()
return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs,
backend='threading' )(delayed(self.get_features )(x, y, s)
 for s, x, y in tqdm_notebook(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list )
df_train = df_train.replace(-1, np.NaN)
 d_median = df_train.median(axis=0)
 d_mean = df_train.mean(axis=0)
 df_train = df_train.fillna(-1)
 dcol = [c for c in df_train.columns if c not in ['id','target']]
df_train['ps_car_13_x_ps_reg_03'] = df_train['ps_car_13'] * df_train['ps_reg_03']
for c in dcol:
if '_bin' not in c:
df_train[c+str('_median_range')] =(df_train[c].values > d_median[c] ).astype(np.int)
 df_train[c+str('_mean_range')] =(df_train[c].values > d_mean[c] ).astype(np.int)
 df_train[c+str('_sq')] = np.power(df_train[c].values,2 ).astype(np.float32)
 df_train[c+str('_log')] = np.log(np.abs(df_train[c].values)+ 1)
 df_train[c+str('_exp')] = np.exp(df_train[c].values)- 1<data_type_conversions>
training_fg = FeatureGenerator(dtype='train', n_jobs=20, chunk_size=150000)
 training_data = training_fg.generate()
test_fg = FeatureGenerator(dtype='test', n_jobs=20, chunk_size=150000)
 test_data = test_fg.generate()
X = training_data.drop(['target', 'seg_id'], axis=1)
 X_test = test_data.drop(['target', 'seg_id'], axis=1)
 test_segs = test_data.seg_id
y = training_data.target
change_datatype(df_train )<split>
np.abs(X.corrwith(y)).sort_values(ascending=False ).head(12 )
features= [c for c in df_train.columns.values if c not in ['id', 'target']]
X_train, X_test, y_train, y_test = train_test_split(df_train[features], df_train['target'], test_size=0.33, random_state=42)
 X_train.head()<train_model>
means_dict = {}
for col in X.columns:
if X[col].isnull().any() :
print(col)
 mean_value = X.loc[X[col] != -np.inf, col].mean()
X.loc[X[col] == -np.inf, col] = mean_value
X[col] = X[col].fillna(mean_value)
 means_dict[col] = mean_value
pipeline = Pipeline([ 
('classifier', RandomForestClassifier(random_state = 42))
])
 pipeline.fit(X_train, y_train)
 preds = pipeline.predict(X_test)
 np.mean(preds == y_test )<define_search_space>
for col in X_test.columns:
if X_test[col].isnull().any() :
X_test.loc[X_test[col] == -np.inf, col] = means_dict[col]
X_test[col] = X_test[col].fillna(means_dict[col] )
hyperparameters = {
'classifier__n_estimators': [20, 30, 50],
'classifier__max_depth': [2, 4],
'classifier__min_samples_leaf': [2, 4]
}
clf = GridSearchCV(pipeline, hyperparameters, cv = 3)
 clf.fit(X_train, y_train )<find_best_params>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )
clf.best_params_<predict_on_test>
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if type(X)== np.ndarray:
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=10000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores
clf.refit
preds = clf.predict(X_test)
 probs = clf.predict_proba(X_test)
 np.mean(preds == y_test )<data_type_conversions>
params = {'num_leaves': 128,
'min_data_in_leaf': 79,
'objective': 'gamma',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": 0.8126672064208567,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501,
'feature_fraction': 0.2
}
oof_lgb, prediction_lgb, feature_importance = train_model(X,
X_test,
y,
params=params,
model_type='lgb',
plot_feature_importance=True )
df_test = df_test.replace(-1, np.NaN)
 dt_median = df_test.median(axis=0)
 dt_mean = df_test.mean(axis=0)
 df_test = df_test.fillna(-1)
 dtcol = [c for c in df_test.columns if c not in ['id']]
df_test['ps_car_13_x_ps_reg_03'] = df_test['ps_car_13'] * df_test['ps_reg_03']
for c in dtcol:
if '_bin' not in c:
df_test[c+str('_median_range')] =(df_test[c].values > dt_median[c] ).astype(np.int)
 df_test[c+str('_mean_range')] =(df_test[c].values > dt_mean[c] ).astype(np.int)
 df_test[c+str('_sq')] = np.power(df_test[c].values,2 ).astype(np.float32)
 df_test[c+str('_log')] = np.log(np.abs(df_test[c].values)+ 1)
 df_test[c+str('_exp')] = np.exp(df_test[c].values)- 1<data_type_conversions>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission['time_to_failure'] = prediction_lgb
print(submission.head())
 submission.to_csv('submission.csv' )
change_datatype(df_test )<save_to_csv>
X.to_csv('train_features.csv', index=False)
 X_test.to_csv('test_features.csv', index=False)
 pd.DataFrame(y ).to_csv('y.csv', index=False )