Kaggle competition: LANL Earthquake Prediction
def fe_step0(actual):
actual['norm_quat'] =(actual['orientation_X']**2 + actual['orientation_Y']**2 + actual['orientation_Z']**2 + actual['orientation_W']**2)
 actual['mod_quat'] =(actual['norm_quat'])**0.5
actual['norm_X'] = actual['orientation_X'] / actual['mod_quat']
actual['norm_Y'] = actual['orientation_Y'] / actual['mod_quat']
actual['norm_Z'] = actual['orientation_Z'] / actual['mod_quat']
actual['norm_W'] = actual['orientation_W'] / actual['mod_quat']
return actual<train_model>
%%time
train_df = pd.read_csv(".. /input/train.csv", dtype={'acoustic_data': 'int16', 'time_to_failure': 'float64'})
 gc.collect()
X_train = fe_step0(X_train)
 X_pred = fe_step0(X_pred)
 print(X_train.shape)
 X_train.head()<concatenate>
seg_lst = list(os.listdir(".. /input/test/"))
for f in seg_lst[:5]:
seg_df = pd.read_csv(f".. /input/test/{f}")
 seg_df.plot()
plt.title(f"{f}")
 plt.show()
plot_transformed(seg_df.acoustic_data)
 del seg_df
gc.collect()
def fe_step1(actual):

 x, y, z, w = actual['norm_X'].tolist() , actual['norm_Y'].tolist() , actual['norm_Z'].tolist() , actual['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
return actual<train_model>
c = 0
for f in seg_lst:
seg_df = pd.read_csv(f".. /input/test/{f}")
 if seg_df.max().values > 2000:
print(f"Huge fluctuations in {f}")
 seg_df.plot()
plt.title(f"{f}")
 plt.show()
plot_transformed(seg_df.acoustic_data)
 c += 1
del seg_df
if c >= 5:
break
gc.collect()
X_train = fe_step1(X_train)
 X_pred = fe_step1(X_pred)
 print(X_train.shape)
 X_train.head()<feature_engineering>
del train_df
gc.collect()
def mean_abs_change(x):
return np.mean(np.abs(np.diff(x)) )<create_dataframe>
derived_trn = pd.DataFrame()
train_rd = pd.read_csv(".. /input/train.csv", iterator=True, chunksize=150000)
 def get_features(chunk):
curr_df = pd.DataFrame()
curr_df["mean"] = [chunk.acoustic_data.mean() ]
curr_df["std"] = [chunk.acoustic_data.std() ]
curr_df["kurtosis"] = [chunk.acoustic_data.kurtosis() ]
curr_df["skew"] = [chunk.acoustic_data.skew() ]
curr_df["quantile_05"], curr_df["quantile_25"], curr_df["quantile_75"], curr_df["quantile_95"] = \
[[x] for x in chunk.acoustic_data.quantile([0.05, 0.25, 0.75, 0.95])]
win_width = 100
slepian_width = 51
windowed = chunk.acoustic_data.rolling(win_width, win_type='slepian' ).mean(width=slepian_width ).dropna()
curr_df[f"window_{win_width}_mean"] = [windowed.mean() ]
curr_df[f"window_[win_width]_std"] = [windowed.std() ]
fft = np.fft.fft(chunk.acoustic_data)
 curr_df["fft_real_mean"] = [fft.real.mean() ]
curr_df["fft_image_mean"] = [fft.imag.mean() ]
curr_df["fft_real_std"] = [fft.real.std() ]
curr_df["fft_image_std"] = [fft.imag.std() ]
sig_first = chunk.acoustic_data.iloc[:20000]
curr_df["mean_first"] = [sig_first.mean() ]
curr_df["std_first"] = [sig_first.std() ]
curr_df["kurtosis_first"] = [sig_first.kurtosis() ]
curr_df["skew_first"] = [sig_first.skew() ]
curr_df["quantile_05_first"], curr_df["quantile_25_first"], \
curr_df["quantile_75_first"], curr_df["quantile_95_first"] = [[x] for x in sig_first.quantile([0.05, 0.25, 0.75, 0.95])]
windowed_first = sig_first.rolling(win_width, win_type='slepian' ).mean(width=slepian_width ).dropna()
curr_df[f"window_{win_width}_mean_first"] = [windowed_first.mean() ]
curr_df[f"window_{win_width}_std_first"] = [windowed_first.std() ]
fft_first = np.fft.fft(sig_first)
 curr_df["fft_real_mean_first"] = [fft_first.real.mean() ]
curr_df["fft_image_mean_first"] = [fft_first.imag.mean() ]
curr_df["fft_real_std_first"] = [fft_first.real.std() ]
curr_df["fft_image_std_first"] = [fft_first.imag.std() ]
sig_last = chunk.acoustic_data.iloc[-20000:]
curr_df["mean_last"] = [sig_last.mean() ]
curr_df["std_last"] = [sig_last.std() ]
curr_df["kurtosis_last"] = [sig_last.kurtosis() ]
curr_df["skew_last"] = [sig_last.skew() ]
curr_df["quantile_05_last"], curr_df["quantile_25_last"], \
curr_df["quantile_75_last"], curr_df["quantile_95_last"] = [[x] for x in sig_last.quantile([0.05, 0.25, 0.75, 0.95])]
windowed_last = sig_last.rolling(win_width, win_type='slepian' ).mean(width=slepian_width ).dropna()
curr_df[f"window_{win_width}_mean_last"] = [windowed_last.mean() ]
curr_df[f"window_{win_width}_std_last"] = [windowed_last.std() ]
fft_last = np.fft.fft(sig_last)
 curr_df["fft_real_mean_last"] = [fft_last.real.mean() ]
curr_df["fft_image_mean_last"] = [fft_last.imag.mean() ]
curr_df["fft_real_std_last"] = [fft_last.real.std() ]
curr_df["fft_image_std_last"] = [fft_last.imag.std() ]
return curr_df
for chunk in train_rd:
curr_df = get_features(chunk)
 curr_df["target"] = chunk.time_to_failure.iloc[-1]
derived_trn = pd.concat([derived_trn, curr_df], ignore_index=True)
 gc.collect()
def get_features(df):
result_df = pd.DataFrame()
for col in df.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
result_df['{}_mean'.format(col)] = df.groupby(['series_id'])[col].mean()
result_df['{}_max'.format(col)] = df.groupby(['series_id'])[col].max()
result_df['{}_min'.format(col)] = df.groupby(['series_id'])[col].min()
result_df['{}_sum'.format(col)] = df.groupby(['series_id'])[col].sum()
result_df['{}_mean_abs_change'.format(col)] = df.groupby(['series_id'])[col].apply(mean_abs_change)
 return result_df<feature_engineering>
derived_trn.to_csv("derived_train.csv", index=False)
 display(derived_trn.head())
 display(derived_trn.loc[(derived_trn.target <= 0.33)&(derived_trn.target >= 0.3)].head())
 features = [col for col in derived_trn.columns if col != "target"]
train_df = get_features(X_train)
 pred_df = get_features(X_pred )<merge>
test_seg_list = list(os.listdir(".. /input/test/"))
derived_test = pd.DataFrame()
for csv_file in test_seg_list:
curr_test_df = pd.read_csv(f".. /input/test/{csv_file}", dtype={"acoustic_data": "int16"})
 curr_df = get_features(curr_test_df)
 curr_df["seg_id"] = [csv_file.split(".")[0]]
derived_test = pd.concat([derived_test, curr_df], ignore_index=True)
 gc.collect()
train_df = train_df.merge(Y_train, on='series_id', how='inner' )<count_values>
derived_test.to_csv("derived_test.csv", index=False)
 display(derived_test.head() )
surfaces =(train_df['surface'].value_counts() ).index<define_variables>
params = dict(
 objective='regression_l1',
n_estimators=5000, learning_rate=0.01,
num_leaves=31, max_depth=-1,
reg_alpha=0.3, reg_lambda=0.5,
metric='mae', 
)
 def augment(df, threshold=None, repeat_times=1):
df = df.copy()
for _ in range(repeat_times):
df_copy =(df.loc[df["target"] <= threshold].copy()
if threshold is not None
else df.copy())
 df = pd.concat([df, df_copy], ignore_index=True)
 df = df.sample(frac=1, random_state=42)
 return df
features = []
for feature in train_df.columns.values[1:-2]:
if 'euler' in feature:
features.append(feature )<categorify>
param_grid = dict(
 n_estimators=[1000], learning_rate=[0.01], 
)
 def search_params(estimator, param_grid, params):
ori_n_estimators = params["n_estimators"]
gscv = GridSearchCV(estimator, cv=5, param_grid=param_grid, scoring='neg_mean_absolute_error', verbose=3)
 best = gscv.fit(derived_trn[features], derived_trn["target"])
 params.update(gscv.best_params_)
 params["n_estimators"] = ori_n_estimators
print(gscv.best_params_)
 return params
label_encoder = LabelEncoder()
train_df['surface'] = label_encoder.fit_transform(train_df['surface'] )<drop_column>
n_splits = 5
n_repeats = 25
rskf = RepeatedStratifiedKFold(n_repeats=n_repeats, n_splits=n_splits, random_state=42)
 predictions = pd.DataFrame()
feature_importances = pd.DataFrame()
oof = np.zeros(( len(derived_trn), n_repeats))
for nf,(trn_idx, val_idx)in enumerate(rskf.split(derived_trn,(derived_trn['target'] <= 1.2))):
trn_features, trn_targets = derived_trn[features].iloc[trn_idx], derived_trn["target"].iloc[trn_idx]
val_features, val_targets = derived_trn[features].iloc[val_idx], derived_trn["target"].iloc[val_idx]
print("
Nf: {}, train on {} samples, val on {} samples".format(nf, len(trn_targets), len(val_targets)))
 model = LGBMRegressor(**params)
 gc.collect()
fit_params = dict(
 eval_set=[(trn_features, trn_targets),(val_features, val_targets)],
verbose=1000,
early_stopping_rounds=1500, 
)
 model.fit(trn_features, trn_targets, **fit_params)
 curr_importance = pd.DataFrame({"feature": features, "importance": model.feature_importances_})
 feature_importances = pd.concat([feature_importances, curr_importance])
 oof[val_idx, nf // n_splits] = model.predict(val_features)
 curr_test_pred = model.predict(derived_test[features])
 predictions[f"pred_{nf}"] = curr_test_pred
<prepare_x_and_y><EOS> 
submission_y = predictions.mean(axis=1 ).values
submission = pd.DataFrame({"seg_id": derived_test["seg_id"], "time_to_failure": submission_y})
 submission.to_csv("submission.csv", index=False)
 display(submission.head())
 display(submission.tail())
 print("Minimum time_to_failure in submission:", submission["time_to_failure"].min() , "sec." )