Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<choose_model_class>
%matplotlib inline
pd.options.display.precision = 8
warnings.filterwarnings("ignore")
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,target['surface'].values)) :
model = RandomForestClassifier(n_estimators=500)
 model.fit(data.iloc[trn_idx],target['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],target['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],target['surface'][val_idx])))
 importances = model.feature_importances_
indices = np.argsort(importances)
 features = data.columns
if model.score(data.iloc[val_idx],target['surface'][val_idx])> 0.91000:
hm = 30
plt.figure(figsize=(7, 10))
plt.title('Feature Importances')
 plt.barh(range(len(indices[:hm])) , importances[indices][:hm], color='b', align='center')
 plt.yticks(range(len(indices[:hm])) , [features[i] for i in indices])
 plt.xlabel('Relative Importance')
 plt.show()
gc.collect()<compute_test_metric>
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )
print('Avg Accuracy', score / folds.n_splits )<compute_test_metric>
index_ocur = train[train['time_to_failure'].diff(-1)<0].index
print(index_ocur )
confusion_matrix(measured,target['surface'] )<save_to_csv>
index_ocur_array = np.array(index_ocur.tolist())
 index_ocur_array
ss['surface'] = le.inverse_transform(predicted.argmax(axis=1))
ss.to_csv('submission.csv', index=False)
 ss.head()<load_from_csv>
del index_ocur
gc.collect()
<import_modules>
intervals = [index_ocur_array[0]]+[index_ocur_array[i]-index_ocur_array[i-1] for i in range(1,len(index_ocur_array)) ]\
+[train.shape[0]-index_ocur_array[-1]]
intervals
import numpy as np
import pandas as pd
import math
import multiprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
import functools
from concurrent.futures import ProcessPoolExecutor
from sklearn.model_selection import KFold<load_from_csv>
index_beg = [0] + list(np.array(index_ocur_array)+1)
 blocks = [(index_beg[i], int(intervals[i]/150000)) for i in range(len(intervals)) ]
blocks
train = pd.read_csv('.. /input/X_train.csv')
 test = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 submit = pd.read_csv('.. /input/sample_submission.csv' )<normalization>
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 total_mean = train['acoustic_data'].mean()
total_std = train['acoustic_data'].std()
total_max = train['acoustic_data'].max()
total_min = train['acoustic_data'].min()
total_sum = train['acoustic_data'].sum()
total_abs_sum = np.abs(train['acoustic_data'] ).sum()
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
 segment = 0
for(index_start, blocknum)in tqdm_notebook(blocks):
for block in tqdm_notebook(range(blocknum)) :
seg = train.iloc[index_start+block*rows:index_start+block*rows+rows]
x = pd.Series(seg['acoustic_data'].values)
 y = seg['time_to_failure'].values[-1]
data_fft = abs(fft(x)) [range(1,int(len(x)/2)) ]/len(x)
 y_tr.loc[segment, 'time_to_failure'] = y
X_tr.loc[segment, 'mean'] = x.mean()
X_tr.loc[segment, 'std'] = x.std()
X_tr.loc[segment, 'max'] = x.max()
X_tr.loc[segment, 'min'] = x.min()
X_tr.loc[segment, 'fft_mean'] = data_fft.mean()
X_tr.loc[segment, 'fft_std'] = data_fft.std()
X_tr.loc[segment, 'fft_max'] = data_fft.max()
X_tr.loc[segment, 'fft_min'] = data_fft.min()
X_tr.loc[segment, 'mean_change_abs'] = np.mean(np.diff(x))
X_tr.loc[segment, 'mean_change_rate'] = calc_change_rate(x)
 X_tr.loc[segment, 'abs_max'] = np.abs(x ).max()
X_tr.loc[segment, 'abs_min'] = np.abs(x ).min()
X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()
X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()
X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()
X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()
X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()
X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()
X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()
X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()
X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()
X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()
X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()
X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()
X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()
X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()
X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()
X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()
X_tr.loc[segment, 'fft_std_first_50000'] = data_fft[:50000].std()
X_tr.loc[segment, 'fft_std_last_50000'] = data_fft[-50000:].std()
X_tr.loc[segment, 'fft_std_first_10000'] = data_fft[:10000].std()
X_tr.loc[segment, 'fft_std_last_10000'] = data_fft[-10000:].std()
X_tr.loc[segment, 'fft_avg_first_50000'] = data_fft[:50000].mean()
X_tr.loc[segment, 'fft_avg_last_50000'] = data_fft[-50000:].mean()
X_tr.loc[segment, 'fft_avg_first_10000'] = data_fft[:10000].mean()
X_tr.loc[segment, 'fft_avg_last_10000'] = data_fft[-10000:].mean()
X_tr.loc[segment, 'fft_min_first_50000'] = data_fft[:50000].min()
X_tr.loc[segment, 'fft_min_last_50000'] = data_fft[-50000:].min()
X_tr.loc[segment, 'fft_min_first_10000'] = data_fft[:10000].min()
X_tr.loc[segment, 'fft_min_last_10000'] = data_fft[-10000:].min()
X_tr.loc[segment, 'fft_max_first_50000'] = data_fft[:50000].max()
X_tr.loc[segment, 'fft_max_last_50000'] = data_fft[-50000:].max()
X_tr.loc[segment, 'fft_max_first_10000'] = data_fft[:10000].max()
X_tr.loc[segment, 'fft_max_last_10000'] = data_fft[-10000:].max()
X_tr.loc[segment, 'max_to_min'] = x.max() / np.abs(x.min())
 X_tr.loc[segment, 'max_to_min_diff'] = x.max() - np.abs(x.min())
 X_tr.loc[segment, 'count_big'] = len(x[np.abs(x)> 500])
 X_tr.loc[segment, 'sum'] = x.sum()
X_tr.loc[segment, 'mean_change_rate_first_50000'] = calc_change_rate(x[:50000])
 X_tr.loc[segment, 'mean_change_rate_last_50000'] = calc_change_rate(x[-50000:])
 X_tr.loc[segment, 'mean_change_rate_first_10000'] = calc_change_rate(x[:10000])
 X_tr.loc[segment, 'mean_change_rate_last_10000'] = calc_change_rate(x[-10000:])
 X_tr.loc[segment, 'q95'] = np.quantile(x, 0.95)
 X_tr.loc[segment, 'q99'] = np.quantile(x, 0.99)
 X_tr.loc[segment, 'q05'] = np.quantile(x, 0.05)
 X_tr.loc[segment, 'q01'] = np.quantile(x, 0.01)
 X_tr.loc[segment, 'fft_q95'] = np.quantile(data_fft, 0.95)
 X_tr.loc[segment, 'fft_q99'] = np.quantile(data_fft, 0.99)
 X_tr.loc[segment, 'fft_q05'] = np.quantile(data_fft, 0.05)
 X_tr.loc[segment, 'fft_q01'] = np.quantile(data_fft, 0.01)
 X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x), 0.95)
 X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x), 0.99)
 X_tr.loc[segment, 'abs_q05'] = np.quantile(np.abs(x), 0.05)
 X_tr.loc[segment, 'abs_q01'] = np.quantile(np.abs(x), 0.01)
 X_tr.loc[segment, 'trend'] = add_trend_feature(x)
 X_tr.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_tr.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_tr.loc[segment, 'abs_std'] = np.abs(x ).std()
X_tr.loc[segment, 'mad'] = x.mad()
X_tr.loc[segment, 'kurt'] = x.kurtosis()
X_tr.loc[segment, 'skew'] = x.skew()
X_tr.loc[segment, 'med'] = x.median()
X_tr.loc[segment, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X_tr.loc[segment, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X_tr.loc[segment, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
X_tr.loc[segment, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
X_tr.loc[segment, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
X_tr.loc[segment, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
X_tr.loc[segment, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
X_tr.loc[segment, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
X_tr.loc[segment, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
X_tr.loc[segment, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
X_tr.loc[segment, 'Moving_average_700_mean'] = x.rolling(window=700 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X_tr.loc[segment, 'exp_Moving_average_300_mean'] =(ewma(x, span=300 ).mean() ).mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_30000_mean'] = ewma(x, span=30000 ).mean().mean(skipna=True)
 no_of_std = 3
X_tr.loc[segment, 'MA_700MA_std_mean'] = x.rolling(window=700 ).std().mean()
X_tr.loc[segment,'MA_700MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_700MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_400MA_std_mean'] = x.rolling(window=400 ).std().mean()
X_tr.loc[segment,'MA_400MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_400MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_1000MA_std_mean'] = x.rolling(window=1000 ).std().mean()
X_tr.drop('Moving_average_700_mean', axis=1, inplace=True)
 X_tr.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_tr.loc[segment, 'q999'] = np.quantile(x,0.999)
 X_tr.loc[segment, 'q001'] = np.quantile(x,0.001)
 X_tr.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
segment+=1
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def feature_engineering(df):
df_new = pd.DataFrame()
df['total_angular_velocity'] = np.sqrt(df['angular_velocity_X'] ** 2 + df['angular_velocity_Y'] ** 2 + df['angular_velocity_Z'] ** 2)
 df['total_linear_acceleration'] = np.sqrt(df['linear_acceleration_X'] ** 2 + df['linear_acceleration_Y'] ** 2 + df['linear_acceleration_Z'] ** 2)
 df['total_orientation'] = np.sqrt(df['orientation_X'] ** 2 + df['orientation_Y'] ** 2 + df['orientation_Z'] ** 2 + df['orientation_W'] ** 2)
 df['linear_acceleration_X_abs'] = df['linear_acceleration_X'].where(df['linear_acceleration_X']>=0, - df['linear_acceleration_X'])
 df['linear_acceleration_Y_abs'] = df['linear_acceleration_Y'].where(df['linear_acceleration_Y']>=0, - df['linear_acceleration_Y'])
 df['linear_acceleration_Z_abs'] = df['linear_acceleration_Z'].where(df['linear_acceleration_Z']>=0, - df['linear_acceleration_Z'])
 df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
x, y, z, w = df['orientation_X'].tolist() , df['orientation_Y'].tolist() , df['orientation_Z'].tolist() , df['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 df['euler_x'] = nx
df['euler_y'] = ny
df['euler_z'] = nz
df['total_angle'] = np.sqrt(df['euler_x'] ** 2 + df['euler_y'] ** 2 + df['euler_z'] ** 2)
 df['angle_vs_acc'] = df['total_angle'] / df['total_linear_acceleration']
df['angle_vs_vel'] = df['total_angle'] / df['total_angular_velocity']
df['acc_times_vel'] = df['total_linear_acceleration'] * df['total_angular_velocity']
df['angle_times_acc'] = df['total_angle'] * df['total_linear_acceleration']
df['angle_times_vel'] = df['total_angle'] * df['total_angular_velocity']
df['angle_times_vel_times_acc'] = df['total_angle'] * df['total_angular_velocity'] * df['total_linear_acceleration']
def f1(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def f2(x):
return np.mean(np.abs(np.diff(x)))
 for col in df.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
df_new[col + '_mean'] = df.groupby(['series_id'])[col].mean()
df_new[col + '_min'] = df.groupby(['series_id'])[col].min()
df_new[col + '_max'] = df.groupby(['series_id'])[col].max()
df_new[col + '_std'] = df.groupby(['series_id'])[col].std()
df_new[col + '_max_to_min'] = df_new[col + '_max'] / df_new[col + '_min']
df_new[col + '_mean_abs_change'] = df.groupby('series_id')[col].apply(f2)
 df_new[col + '_mean_change_of_abs_change'] = df.groupby('series_id')[col].apply(f1)
 df_new[col + '_abs_max'] = df.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 df_new[col + '_abs_min'] = df.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 return df_new<categorify>
X_tr.isnull().sum().sum()
with multiprocessing.Pool() as pool:
train, test = pool.map(feature_engineering, [train, test] )<prepare_x_and_y>
X_tr = X_tr[:-10]
y_tr = y_tr[:-10]
train = train.reset_index()
test = test.reset_index()<categorify>
np.abs(X_tr.corrwith(y_tr['time_to_failure'])).sort_values(ascending=False ).head(12 )
le = LabelEncoder()
target['surface'] = le.fit_transform(target['surface'] )<compute_train_metric>
means_dict = {}
for col in X_tr.columns:
if X_tr[col].isnull().any() :
print(col)
 mean_value = X_tr.loc[X_tr[col] != -np.inf, col].mean()
X_tr.loc[X_tr[col] == -np.inf, col] = mean_value
X_tr[col] = X_tr[col].fillna(mean_value)
 means_dict[col] = mean_value
def _distance(a, b):
distance = np.linalg.norm(a - b)
 return distance
def _knn_distance(xtr_c, target, k):
distances = np.array([_distance(target, x)for x in xtr_c])
 sorted_distances = np.sort(distances)
 nearest_distances = sorted_distances[:k]
sum_distances = np.sum(nearest_distances)
 return sum_distances
def _knn_distance_fold(xtr_c, k, folds, target):
distances = np.empty([folds])
 kf = KFold(shuffle=True, n_splits=folds)
 for i,(train_index, _)in enumerate(kf.split(xtr_c)) :
xtr_c_sampled = xtr_c[train_index]
distance = _knn_distance(xtr_c_sampled, target, k)
 distances[i] = distance
average_distance = distances.mean()
return average_distance
def knn_extract(xtr, ytr, xte, k=1, folds=5, nprocesses=-1):
if nprocesses == -1:
nprocesses = multiprocessing.cpu_count()
classes = np.unique(ytr)
 features = np.zeros([len(xte), len(classes)* k])
 for i, class_ in enumerate(classes):
xtr_c = xtr[ytr == class_]
for j, k_n in enumerate(range(1, k + 1), 1):
f = functools.partial(_knn_distance_fold, xtr_c, k_n, folds)
 with ProcessPoolExecutor(max_workers=nprocesses)as executor:
feature = executor.map(f, xte)
 features[:, i * j] = list(feature)
 return features<split>
scaler = StandardScaler()
scaler.fit(X_tr)
 X_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns )
train_array = train.values
test_array = test.values
knn_train = knn_extract(train_array, target['surface'].values, train_array)
 knn_test = knn_extract(train_array, target['surface'].values, test_array )<create_dataframe>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)
 plt.figure(figsize=(22, 16))
for i, seg_id in enumerate(tqdm_notebook(X_test.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = pd.Series(seg['acoustic_data'].values)
 data_fft = abs(fft(x)) [range(1,int(len(x)/2)) ]/len(x)
 X_test.loc[seg_id, 'mean'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'fft_mean'] = data_fft.mean()
X_test.loc[seg_id, 'fft_std'] = data_fft.std()
X_test.loc[seg_id, 'fft_max'] = data_fft.max()
X_test.loc[seg_id, 'fft_min'] = data_fft.min()
X_test.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(x))
X_test.loc[seg_id, 'mean_change_rate'] = calc_change_rate(x)
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
X_test.loc[seg_id, 'abs_min'] = np.abs(x ).min()
X_test.loc[seg_id, 'std_first_50000'] = x[:50000].std()
X_test.loc[seg_id, 'std_last_50000'] = x[-50000:].std()
X_test.loc[seg_id, 'std_first_10000'] = x[:10000].std()
X_test.loc[seg_id, 'std_last_10000'] = x[-10000:].std()
X_test.loc[seg_id, 'avg_first_50000'] = x[:50000].mean()
X_test.loc[seg_id, 'avg_last_50000'] = x[-50000:].mean()
X_test.loc[seg_id, 'avg_first_10000'] = x[:10000].mean()
X_test.loc[seg_id, 'avg_last_10000'] = x[-10000:].mean()
X_test.loc[seg_id, 'min_first_50000'] = x[:50000].min()
X_test.loc[seg_id, 'min_last_50000'] = x[-50000:].min()
X_test.loc[seg_id, 'min_first_10000'] = x[:10000].min()
X_test.loc[seg_id, 'min_last_10000'] = x[-10000:].min()
X_test.loc[seg_id, 'max_first_50000'] = x[:50000].max()
X_test.loc[seg_id, 'max_last_50000'] = x[-50000:].max()
X_test.loc[seg_id, 'max_first_10000'] = x[:10000].max()
X_test.loc[seg_id, 'max_last_10000'] = x[-10000:].max()
X_test.loc[seg_id, 'fft_std_first_50000'] = data_fft[:50000].std()
X_test.loc[seg_id, 'fft_std_last_50000'] = data_fft[-50000:].std()
X_test.loc[seg_id, 'fft_std_first_10000'] = data_fft[:10000].std()
X_test.loc[seg_id, 'fft_std_last_10000'] = data_fft[-10000:].std()
X_test.loc[seg_id, 'fft_avg_first_50000'] = data_fft[:50000].mean()
X_test.loc[seg_id, 'fft_avg_last_50000'] = data_fft[-50000:].mean()
X_test.loc[seg_id, 'fft_avg_first_10000'] = data_fft[:10000].mean()
X_test.loc[seg_id, 'fft_avg_last_10000'] = data_fft[-10000:].mean()
X_test.loc[seg_id, 'fft_min_first_50000'] = data_fft[:50000].min()
X_test.loc[seg_id, 'fft_min_last_50000'] = data_fft[-50000:].min()
X_test.loc[seg_id, 'fft_min_first_10000'] = data_fft[:10000].min()
X_test.loc[seg_id, 'fft_min_last_10000'] = data_fft[-10000:].min()
X_test.loc[seg_id, 'fft_max_first_50000'] = data_fft[:50000].max()
X_test.loc[seg_id, 'fft_max_last_50000'] = data_fft[-50000:].max()
X_test.loc[seg_id, 'fft_max_first_10000'] = data_fft[:10000].max()
X_test.loc[seg_id, 'fft_max_last_10000'] = data_fft[-10000:].max()
X_test.loc[seg_id, 'max_to_min'] = x.max() / np.abs(x.min())
 X_test.loc[seg_id, 'max_to_min_diff'] = x.max() - np.abs(x.min())
 X_test.loc[seg_id, 'count_big'] = len(x[np.abs(x)> 500])
 X_test.loc[seg_id, 'sum'] = x.sum()
X_test.loc[seg_id, 'mean_change_rate_first_50000'] = calc_change_rate(x[:50000])
 X_test.loc[seg_id, 'mean_change_rate_last_50000'] = calc_change_rate(x[-50000:])
 X_test.loc[seg_id, 'mean_change_rate_first_10000'] = calc_change_rate(x[:10000])
 X_test.loc[seg_id, 'mean_change_rate_last_10000'] = calc_change_rate(x[-10000:])
 X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'fft_q95'] = np.quantile(data_fft, 0.95)
 X_test.loc[seg_id, 'fft_q99'] = np.quantile(data_fft, 0.99)
 X_test.loc[seg_id, 'fft_q05'] = np.quantile(data_fft, 0.05)
 X_test.loc[seg_id, 'fft_q01'] = np.quantile(data_fft, 0.01)
 X_test.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(x), 0.95)
 X_test.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(x), 0.99)
 X_test.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(x), 0.05)
 X_test.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(x), 0.01)
 X_test.loc[seg_id, 'trend'] = add_trend_feature(x)
 X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_test.loc[seg_id, 'abs_mean'] = np.abs(x ).mean()
X_test.loc[seg_id, 'abs_std'] = np.abs(x ).std()
X_test.loc[seg_id, 'mad'] = x.mad()
X_test.loc[seg_id, 'kurt'] = x.kurtosis()
X_test.loc[seg_id, 'skew'] = x.skew()
X_test.loc[seg_id, 'med'] = x.median()
X_test.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X_test.loc[seg_id, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X_test.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
X_test.loc[seg_id, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
X_test.loc[seg_id, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
X_test.loc[seg_id, 'Moving_average_700_mean'] = x.rolling(window=700 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X_test.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(x, span=300 ).mean() ).mean(skipna=True)
 X_test.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000 ).mean().mean(skipna=True)
 X_test.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(x, span=30000 ).mean().mean(skipna=True)
 no_of_std = 3
X_test.loc[seg_id, 'MA_700MA_std_mean'] = x.rolling(window=700 ).std().mean()
X_test.loc[seg_id,'MA_700MA_BB_high_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X_test.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X_test.loc[seg_id,'MA_700MA_BB_low_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X_test.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X_test.loc[seg_id, 'MA_400MA_std_mean'] = x.rolling(window=400 ).std().mean()
X_test.loc[seg_id,'MA_400MA_BB_high_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X_test.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X_test.loc[seg_id,'MA_400MA_BB_low_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X_test.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X_test.loc[seg_id, 'MA_1000MA_std_mean'] = x.rolling(window=1000 ).std().mean()
X_test.drop('Moving_average_700_mean', axis=1, inplace=True)
 X_test.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_test.loc[seg_id, 'q999'] = np.quantile(x,0.999)
 X_test.loc[seg_id, 'q001'] = np.quantile(x,0.001)
 X_test.loc[seg_id, 'ave10'] = stats.trim_mean(x, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_test.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_test.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_test.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_test.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_test.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_test.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_test.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_test.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_test.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_test.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_test.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_test.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_test.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_test.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_test.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_test.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_test.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_test.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
if i < 12:
plt.subplot(6, 4, i + 1)
 plt.plot(seg['acoustic_data'])
 plt.title(seg_id)
knn_train = pd.DataFrame(knn_train)
 knn_test = pd.DataFrame(knn_test)
 knn_train = pd.concat([train, knn_train], axis=1)
 knn_test = pd.concat([test, knn_test], axis=1 )<categorify>
X_test.isnull().sum().sum()
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True)
 knn_train.fillna(0, inplace=True)
 knn_test.fillna(0, inplace=True)
 train.replace(-np.inf, 0, inplace=True)
 train.replace(np.inf, 0, inplace=True)
 test.replace(-np.inf, 0, inplace=True)
 test.replace(np.inf, 0, inplace=True)
 knn_train.replace(-np.inf, 0, inplace=True)
 knn_train.replace(np.inf, 0, inplace=True)
 knn_test.replace(-np.inf, 0, inplace=True)
 knn_test.replace(np.inf, 0, inplace=True )<train_model>
for col in X_test.columns:
if X_test[col].isnull().any() :
X_test.loc[X_test[col] == -np.inf, col] = means_dict[col]
X_test[col] = X_test[col].fillna(means_dict[col])
 X_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns )
folds = StratifiedKFold(n_splits=100, shuffle=True, random_state=546789)
 sub_preds_rf = np.zeros(( knn_test.shape[0], 9))
score = 0
counter = 0
for train_index, test_index in folds.split(knn_train, target['surface']):
clf_knn = RandomForestClassifier(n_estimators=200, n_jobs=-1)
 clf_knn.fit(knn_train.iloc[train_index], target['surface'][train_index])
 sub_preds_rf += clf_knn.predict_proba(knn_test)/ folds.n_splits
score += clf_knn.score(knn_train.iloc[test_index], target['surface'][test_index])
 counter += 1
print('avg accuracy : {}'.format(score / folds.n_splits))<features_selection>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )
fti_knn = clf_knn.feature_importances_
for i, feat in enumerate(knn_train.columns):
print('{0} : {1:>.6f}'.format(feat, fti_knn[i]))<save_to_csv>
def train_model(X=X_train_scaled, X_test=X_test_scaled, y=y_tr, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=10000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction
submit['surface'] = le.inverse_transform(sub_preds_rf.argmax(axis=1))
submit.to_csv('submit_with_knn.csv', index=False )<train_model>
X_train_scaled.isnull().sum().sum()
folds = StratifiedKFold(n_splits=100, shuffle=True, random_state=546789)
 sub_preds_rf = np.zeros(( test.shape[0], 9))
score = 0
counter = 0
for train_index, test_index in folds.split(train, target['surface']):
clf = RandomForestClassifier(n_estimators=200, n_jobs=-1)
 clf.fit(train.iloc[train_index], target['surface'][train_index])
 sub_preds_rf += clf.predict_proba(test)/ folds.n_splits
score += clf.score(train.iloc[test_index], target['surface'][test_index])
 counter += 1
print('avg accuracy : {}'.format(score / folds.n_splits))<save_to_csv>
y_tr.isnull().sum()
submit['surface'] = le.inverse_transform(sub_preds_rf.argmax(axis=1))
submit.to_csv('submit_without_knn.csv', index=False )<import_modules>
params = {'num_leaves': 128,
'min_data_in_leaf': 79,
'objective': 'huber',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": 0.8126672064208567,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501
}
oof_lgb, prediction_lgb, feature_importance = train_model(params=params, model_type='lgb', plot_feature_importance=True )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier<load_from_csv>
xgb_params = {'eta': 0.03,
'max_depth': 9,
'subsample': 0.9,
'objective': 'reg:linear',
'eval_metric': 'mae',
'silent': True,
'nthread': 4}
oof_xgb, prediction_xgb = train_model(X=X_train_scaled, X_test=X_test_scaled, params=xgb_params, model_type='xgb' )
X_train = pd.read_csv('.. /input/X_train.csv')
 Y_train = pd.read_csv('.. /input/y_train.csv')
 X_pred = pd.read_csv('.. /input/X_test.csv' )<concatenate>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z<feature_engineering>
submission['time_to_failure'] =(prediction_lgb+prediction_xgb)/2
print(submission.head())
 submission.to_csv('submission.csv' )