Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<import_modules>
%matplotlib inline
pd.options.display.precision = 15
import scipy as sp
from functools import partial
from collections import Counter<import_modules>
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.model_selection import GroupShuffleSplit
from sklearn.metrics import accuracy_score<set_options>
rows = 150000
segments = int(np.floor(train.shape[0] / rows))
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm_notebook(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x_raw = seg['acoustic_data']
x = x_raw.values
y = seg['time_to_failure'].values[-1]
y_tr.loc[segment, 'time_to_failure'] = y
X_tr.loc[segment, 'ave'] = x.mean()
X_tr.loc[segment, 'std'] = x.std()
X_tr.loc[segment, 'max'] = x.max()
X_tr.loc[segment, 'min'] = x.min()
X_tr.loc[segment, 'q01'] = np.quantile(x,0.01)
 X_tr.loc[segment, 'q05'] = np.quantile(x,0.05)
 X_tr.loc[segment, 'q95'] = np.quantile(x,0.95)
 X_tr.loc[segment, 'q99'] = np.quantile(x,0.99)
 X_tr.loc[segment, 'abs_median'] = np.median(np.abs(x))
X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x),0.95)
 X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x),0.99)
 X_tr.loc[segment, 'F_test'], X_tr.loc[segment, 'p_test'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_tr.loc[segment, 'av_change_abs'] = np.mean(np.diff(x))
X_tr.loc[segment, 'av_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_tr.loc[segment, 'abs_max'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_raw.rolling(windows ).std().dropna().values
x_roll_mean = x_raw.rolling(windows ).mean().dropna().values
X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
warnings.filterwarnings('ignore' )<import_modules>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)
 plt.figure(figsize=(22, 16))
for i, seg_id in enumerate(tqdm_notebook(X_test.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x_raw = seg['acoustic_data']
x_roll = x_raw.rolling(windows ).std().dropna().values
x = x_raw.values
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'abs_median'] = np.median(np.abs(x))
X_test.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(x),0.95)
 X_test.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(x),0.99)
 X_test.loc[seg_id, 'F_test'], X_tr.loc[segment, 'p_test'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_test.loc[seg_id, 'av_change_abs'] = np.mean(np.diff(x))
X_test.loc[seg_id, 'av_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_raw.rolling(windows ).std().dropna().values
x_roll_mean = x_raw.rolling(windows ).mean().dropna().values
X_test.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_test.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_test.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_test.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_test.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_test.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_test.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_test.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_test.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_test.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_test.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_test.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_test.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_test.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_test.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_test.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_test.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_test.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
if i < 12:
plt.subplot(3, 4, i + 1)
 plt.plot(seg['acoustic_data'])
 plt.title(seg_id )
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
from sklearn.svm import SVC<load_from_csv>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )
tr = pd.read_csv('.. /input/X_train.csv')
 te = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv' )<define_variables>
def train_model(X=X_tr, X_test=X_test, y=y_tr, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_tr.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_tr.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'rcv':
model = RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_absolute_error', cv=3)
 model.fit(X_train, y_train)
 print(model.alpha_)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, oof)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction
tr0=tr
te0=te<load_from_csv>
params = {'num_leaves': 64,
'min_data_in_leaf': 50,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.5,
"bagging_freq": 2,
"bagging_fraction": 0.5,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 1.0,
'reg_lambda': 1.0,
}
oof_lgb, prediction_lgb, feature_importance = train_model(params=params, model_type='lgb', plot_feature_importance=True )
df_target=pd.read_csv('.. /input/y_train.csv')
 df_target.head()<categorify>
params = {'num_leaves': 128,
'min_data_in_leaf': 50,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.5,
"bagging_freq": 2,
"bagging_fraction": 0.7,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 2.0,
'reg_lambda': 1.5,
}
oof_lgb2, prediction_lgb2, feature_importance2 = train_model(params=params, model_type='lgb', plot_feature_importance=True )
le=LabelEncoder()
df_target['surface_id']=le.fit_transform(df_target['surface'])
 df_target.head()<groupby>
params = {'num_leaves': 32,
'min_data_in_leaf': 10,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 2,
"bagging_fraction": 0.8,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 1.0,
'reg_lambda': 0.5,
}
oof_lgb3, prediction_lgb3, feature_importance3 = train_model(params=params, model_type='lgb', plot_feature_importance=True )
df_target_s=df_target[['surface','surface_id']].groupby(by='surface' ).max().reset_index()
df_target_s<groupby>
xgb_params = {'eta': 0.01, 'max_depth': 6, 'subsample': 0.8, 'colsample_bytree': 0.8,'colsample_bylevel': 0.8, 'colsample_bynode': 0.8,
'lambda': 0.1, 'alpha' : 0.1, 'objective': 'reg:linear', 'eval_metric': 'mae', 'silent': True, 'nthread': 4}
oof_xgb, prediction_xgb = train_model(params=xgb_params, model_type='xgb' )
df_target_cnt=target.groupby(by='surface' ).count() ['series_id']
df_target_cnt=df_target_cnt.reset_index()<merge>
xgb_params = {'eta': 0.01, 'max_depth': 12, 'subsample': 0.5, 'colsample_bytree': 0.5,'colsample_bylevel': 0.5, 'colsample_bynode': 0.5,
'lambda': 0.5, 'alpha' : 0.5, 'objective': 'reg:linear', 'eval_metric': 'mae', 'silent': True, 'nthread': 4}
oof_xgb2, prediction_xgb2 = train_model(params=xgb_params, model_type='xgb' )
df_target_s=df_target_s.merge(df_target_cnt,on='surface')
 df_target_s<filter>
print(mean_absolute_error(y_tr,(oof_lgb + oof_lgb2 + oof_lgb3 + oof_xgb + oof_xgb2)/5))
df_target[df_target['group_id']==5]<groupby>
submission['time_to_failure'] =(prediction_lgb + prediction_lgb2 + prediction_lgb3 + prediction_xgb + prediction_xgb2)/ 5
print(submission.head())
 submission.to_csv('submission_all.csv' )