Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<import_modules>
warnings.filterwarnings("ignore")
from sklearn.decomposition import PCA<normalization>
scaled_train_X = pd.read_csv('.. /input/all-features/train_X.csv')
 scaled_test_X = pd.read_csv('.. /input/all-features/test_X.csv')
 train_y = pd.read_csv('.. /input/all-features/train_y.csv' )
pca = PCA(n_components=3, svd_solver='full',random_state=1984)
 pca.fit(XC)
 xpc= pca.transform(XC )<set_options>
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.shape
pca.explained_variance_ratio_<prepare_output>
mse_scoring = make_scorer(mean_squared_error, greater_is_better=False)
 k_fold = KFold(n_splits=7, shuffle=True, random_state=13 )
XPC = pd.DataFrame(data = xpc , columns = ['PC1', 'PC2','PC3'])
 XPC['surface_id'] = y
XPC.head()<define_search_space>
def make_objective(model, X, y, space, cv, scoring):
@use_named_args(space)
 def objective(**params):
model.set_params(**params)
 return -np.mean(cross_val_score(model,
X, y,
cv=cv,
n_jobs=-1,
scoring=scoring))
return objective
selected_sufaces=[1,3,5,6,7,8]<categorify>
xgb_bayes_params = {'colsample_bytree': 0.3706219857878677,
'learning_rate': 0.16624226726409647,
'max_bin': 93400,
'max_depth': 134,
'min_child_samples': 22,
'min_child_weight': 4,
'n_estimators': 8028,
'num_leaves': 27,
'reg_alpha': 1.081049236893711e-05,
'reg_lambda': 1.043686239159047,
'scale_pos_weight': 0.19222548462579486,
'subsample': 0.6941640075502717,
'subsample_for_bin': 375140,
'subsample_freq': 7}
params=xgb_bayes_params
pcat=pca
tpc=pcat.transform(TC )<prepare_output>
counter = 0
xgb_model = xgb.XGBRegressor(
 metric='mae',
n_jobs=1,
verbose=0 
)
 dimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),
Integer(2, 500, name='num_leaves'),
Integer(0, 500, name='max_depth'),
Integer(0, 200, name='min_child_samples'),
Integer(100, 100000, name='max_bin'),
Real(0.01, 1.0, 'uniform', name='subsample'),
Integer(0, 10, name='subsample_freq'),
Real(0.01, 1.0, 'uniform', name='colsample_bytree'),
Integer(0, 10, name='min_child_weight'),
Integer(100000, 500000, name='subsample_for_bin'),
Real(1e-9, 1000, 'log-uniform', name='reg_lambda'),
Real(1e-9, 1.0, 'log-uniform', name='reg_alpha'),
Real(1e-6, 500, 'log-uniform', name='scale_pos_weight'),
Integer(10, 10000, name='n_estimators')]
objective = make_objective(xgb_model,
scaled_train_X, train_y,
space=dimensions,
cv=k_fold,
scoring=mse_scoring )
TPC = pd.DataFrame(data = tpc , columns = ['PC1','PC2','PC3'])
 TPC['surface_id'] = preds
TPC.head()<prepare_output>
PandasDataFrame = TypeVar('pandas.core.frame.DataFrame')
 def run_xgb(
 X_tr: PandasDataFrame,
X_val: PandasDataFrame,
y_tr: PandasDataFrame,
y_val: PandasDataFrame,
test_data: PandasDataFrame,
params: Dict 
):

 early_stop = 200
num_rounds = 10000
verbose_eval=1000
d_train = xgb.DMatrix(X_tr.values, label = y_tr)
 d_valid = xgb.DMatrix(X_val.values, label = y_val)
 d_test = xgb.DMatrix(test_data.values)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist,
verbose_eval=verbose_eval,early_stopping_rounds=early_stop)
 imps=model.get_score(importance_type='gain')
 f_nums = model.feature_names
f_names = list(scaled_train_X)
 f_dict = dict(zip(f_nums,f_names))
keys=[]
for i in imps:
keys.append(f_dict.get(i))
X_tr_new = X_tr.loc[:,keys]
X_val_new=X_val.loc[:,keys]
test_data_new=test_data.loc[:,keys]
d_train = xgb.DMatrix(X_tr_new.values, label = y_tr)
 d_valid = xgb.DMatrix(X_val_new.values, label = y_val)
 d_test = xgb.DMatrix(test_data_new.values)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist,
verbose_eval=verbose_eval,early_stopping_rounds=early_stop)
 val_pred = model.predict(d_valid)
 prediction = model.predict(d_test)
 return val_pred, prediction
def run_cv_model(
 train_data,
train_target,
test_data,
model_fn,
params,
eval_fn=None,
label='model',
feature_imp=False,
n_folds = 5 
):
oof_val = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
oof_predict = np.zeros(( n_folds, test_data.shape[0]))
scores = []
folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 train_columns = train_data.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data, train_target.values)) :
strLog = "fold {} for {}".format(fold_, label)
 print(strLog)
 X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]
y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]
val_pred, prediction= model_fn(
 X_tr, X_val,
y_tr, y_val,
test_data,
params 
)
 score = mean_squared_error(y_val, val_pred)
 scores.append(score)
 oof_val[val_idx] = val_pred
if feature_imp == True:
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = feature_importances[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof_predict[fold_] = prediction
predictions += prediction/folds.n_splits
print('CV score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(score), np.std(score)))
 print('CV mean score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(scores), np.std(scores)))
 return oof_val, oof_predict, predictions
coh = pd.DataFrame()
coh['prev_surface_id']=preds
coh['prev_group_id']=TC['group_id']
coh['prev_id']=coh.index
coh.head()<groupby>
fix_xgb_params = {
"boosting": "gblinear",
"metric": 'mae',
}
xgb_params = {**fix_xgb_params, **xgb_bayes_params}
oof_val, oof_predict, predictions = run_cv_model(
 train_data=scaled_train_X,
train_target=train_y,
test_data=scaled_test_X,
model_fn=run_xgb,
params=xgb_params,
eval_fn=None,
label="xgb",
feature_imp=False,
n_folds=10 
)
<merge><EOS> 
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )