Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<filter>
warnings.filterwarnings("ignore")
 PandasDataFrame = TypeVar('pandas.core.frame.DataFrame')
%%time
for i in range(0,len(prev_merge)) :
gr=prev_merge['group_id'].loc[i]
prev_gr=prev_merge['prev_group_id'].loc[i]
if(gr==-1):
prev_merge['surface_id'].loc[i]=gs['surface_id'][gs['group_id']==prev_gr].values[0]<data_type_conversions>
train_X = pd.read_csv('.. /input/all-features/train_X.csv')
 test_X = pd.read_csv('.. /input/all-features/test_X.csv')
 train_y = pd.read_csv('.. /input/all-features/train_y.csv' )
prev_merge=prev_merge.astype(int)
 prev_merge.head()<feature_engineering>
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.shape
prev_merge['final_surface_id']=prev_merge['prev_surface_id']
prev_merge.head()<statistical_test>
mse_scoring = make_scorer(mean_squared_error, greater_is_better=False)
 mae_scoring = make_scorer(mean_absolute_error, greater_is_better=False)
def counter_distance(cnt,ref):
vals=cnt.values()
tot=sum(vals)
 cnt=sorted(cnt.items())
 dist=0
for i in range(0,len(cnt)) :
dist+=np.abs(ref[i]-cnt[i][1]/tot)
 return dist<statistical_test>
def run_cat(
 X_tr: PandasDataFrame,
X_val: PandasDataFrame,
y_tr: PandasDataFrame,
y_val: PandasDataFrame,
test_data: PandasDataFrame,
params: Dict 
):

 early_stop = 200
num_rounds = 10000
verbose_eval=1000
d_train = cat.Pool(X_tr, label=y_tr)
 d_valid = cat.Pool(X_val, label=y_val)
 d_predict = cat.Pool(test_data)
 cat_model = cat.CatBoostRegressor(**params)
 cat_model.fit(
 X=d_train,use_best_model=True,
eval_set=d_valid,
early_stopping_rounds=early_stop 
)
 val_pred = cat_model.predict(d_valid)
 prediction = cat_model.predict(d_predict)
 return val_pred, prediction, cat_model.get_feature_importance()
def run_cv_model(
 train_data,
train_target,
test_data,
model_fn, params,
scoring,
label='model',
feature_imp=False,
n_folds = 8 
):
oof_val = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
oof_predict = np.zeros(( n_folds, test_data.shape[0]))
scores = []
feature_importance_df = pd.DataFrame()
folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 train_columns = train_data.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data, train_target.values)) :
strLog = "fold {} for {}".format(fold_, label)
 print(strLog)
 X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]
y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]
val_pred, prediction, feature_importances = model_fn(
 X_tr, X_val,
y_tr, y_val,
test_data,
params 
)
 score = scoring(y_val, val_pred)
 scores.append(score)
 oof_val[val_idx] = val_pred
if feature_imp == True:
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = feature_importances[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof_predict[fold_] = prediction
predictions += prediction/folds.n_splits
print('CV score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(score), np.std(score)))
 print('CV mean score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(scores), np.std(scores)))
 return oof_val, oof_predict, predictions, feature_importance_df, scores
d_freq=[]
corrections=0
for i in range(0,len(prev_merge)) :
gr=prev_merge['group_id'].loc[i]
if(gr==-1):
hp1=prev_merge['final_surface_id']
hp2=hp1.copy()
hp2[i]=prev_merge['surface_id'].loc[i]
d_hp1=counter_distance(Counter(hp1),attended_prc)
 d_hp2=counter_distance(Counter(hp2),attended_prc)
 if(d_hp2<d_hp1):
d_freq=d_freq+[d_hp2]
corrections=corrections+1
prev_merge['final_surface_id'] = hp2
else:
d_freq=d_freq+[d_hp1]
print('Corrections made=',corrections )<count_values>
def cat_bayesian(**cat_bayes_params):
cat_bayes_params['iterations'] = int(cat_bayes_params['iterations'])
 cat_bayes_params['depth'] = int(cat_bayes_params['depth'])
 cat_bayes_params['border_count'] = int(cat_bayes_params['border_count'])
 cat_bayes_params['l2_leaf_reg'] = int(cat_bayes_params['l2_leaf_reg'])
 cat_bayes_params['one_hot_max_size'] = int(cat_bayes_params['one_hot_max_size'])
 cat_bayes_params['max_ctr_complexity'] = int(cat_bayes_params['max_ctr_complexity'])
 cat_bayes_params['random_seed'] = int(cat_bayes_params['random_seed'])
 fix_cat_params = {
'task_type':"GPU",
'loss_function':'MAE',
'eval_metric':'MAE',
'verbose': False
}
cat_params = {**fix_cat_params, **cat_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_X,
train_target=train_y,
test_data=test_X,
model_fn=run_cat,
params=cat_params,
scoring=mean_absolute_error,
label="cat",
feature_imp=True,
n_folds=5 
)
 return -np.mean(scores )
Counter(prev_merge['prev_surface_id'] )<count_values>
search_spaces = {
'iterations':(200, 500),
'depth':(1, 9),
'learning_rate':(0.01, 1.0),
'random_strength':(1e-9, 10),
'bagging_temperature':(0.0, 1.0),
'border_count':(1, 100),
'l2_leaf_reg':(2, 50),
'one_hot_max_size':(0, 20),
'max_ctr_complexity':(1, 3),
'random_seed':(1,81)
 }
cat_bo = BayesianOptimization(cat_bayesian,search_spaces, random_state=9)
Counter(prev_merge['final_surface_id'] )<save_to_csv>
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 cat_bo.maximize(init_points=10, n_iter=10, acq='ucb' )
ss.drop('surface_id',axis=1,inplace=True)
 ss.to_csv('submission.csv', index=False )<categorify>
cat_bayes_params = cat_bo.max['params']
cat_bayes_params['iterations'] = int(cat_bayes_params['iterations'])
 cat_bayes_params['depth'] = int(cat_bayes_params['depth'])
 cat_bayes_params['border_count'] = int(cat_bayes_params['border_count'])
 cat_bayes_params['l2_leaf_reg'] = int(cat_bayes_params['l2_leaf_reg'])
 cat_bayes_params['one_hot_max_size'] = int(cat_bayes_params['one_hot_max_size'])
 cat_bayes_params['max_ctr_complexity'] = int(cat_bayes_params['max_ctr_complexity'])
 cat_bayes_params['random_seed'] = int(cat_bayes_params['random_seed'] )
%matplotlib inline
le = preprocessing.LabelEncoder()
warnings.filterwarnings('ignore')
 gc.enable()<load_from_csv>
class FeatureSelector() :

 def __init__(self, data, labels=None):
self.data = data
self.labels = labels
if labels is None:
print('No labels provided.Feature importance based methods are not available.')
 self.base_features = list(data.columns)
 self.one_hot_features = None
self.record_missing = None
self.record_single_unique = None
self.record_collinear = None
self.record_zero_importance = None
self.record_low_importance = None
self.missing_stats = None
self.unique_stats = None
self.corr_matrix = None
self.feature_importances = None
self.ops = {}
self.one_hot_correlated = False
def identify_missing(self, missing_threshold):

 self.missing_threshold = missing_threshold
missing_series = self.data.isnull().sum() / self.data.shape[0]
self.missing_stats = pd.DataFrame(missing_series ).rename(columns = {'index': 'feature', 0: 'missing_fraction'})
 self.missing_stats = self.missing_stats.sort_values('missing_fraction', ascending = False)
 record_missing = pd.DataFrame(missing_series[missing_series > missing_threshold] ).reset_index().rename(columns =
{'index': 'feature',
0: 'missing_fraction'})
 to_drop = list(record_missing['feature'])
 self.record_missing = record_missing
self.ops['missing'] = to_drop
print('%d features with greater than %0.2f missing values.
 ' %(len(self.ops['missing']), self.missing_threshold))
def identify_single_unique(self):

 unique_counts = self.data.nunique()
self.unique_stats = pd.DataFrame(unique_counts ).rename(columns = {'index': 'feature', 0: 'nunique'})
 self.unique_stats = self.unique_stats.sort_values('nunique', ascending = True)
 record_single_unique = pd.DataFrame(unique_counts[unique_counts == 1] ).reset_index().rename(columns = {'index': 'feature',
0: 'nunique'})
 to_drop = list(record_single_unique['feature'])
 self.record_single_unique = record_single_unique
self.ops['single_unique'] = to_drop
print('%d features with a single unique value.
 ' % len(self.ops['single_unique']))
def identify_collinear(self, correlation_threshold, one_hot=False):

 self.correlation_threshold = correlation_threshold
self.one_hot_correlated = one_hot
if one_hot:
features = pd.get_dummies(self.data)
 self.one_hot_features = [column for column in features.columns if column not in self.base_features]
self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)
 corr_matrix = pd.get_dummies(features ).corr()
else:
corr_matrix = self.data.corr()
self.corr_matrix = corr_matrix
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
record_collinear = pd.DataFrame(columns = ['drop_feature', 'corr_feature', 'corr_value'])
 for column in to_drop:
corr_features = list(upper.index[upper[column].abs() > correlation_threshold])
 corr_values = list(upper[column][upper[column].abs() > correlation_threshold])
 drop_features = [column for _ in range(len(corr_features)) ]
temp_df = pd.DataFrame.from_dict({'drop_feature': drop_features,
'corr_feature': corr_features,
'corr_value': corr_values})
 record_collinear = record_collinear.append(temp_df, ignore_index = True)
 self.record_collinear = record_collinear
self.ops['collinear'] = to_drop
print('%d features with a correlation magnitude greater than %0.2f.
 ' %(len(self.ops['collinear']), self.correlation_threshold))
def identify_zero_importance(self, feature_importances):

 if self.labels is None:
raise ValueError("No training labels provided.")
 features = pd.get_dummies(self.data)
 self.one_hot_features = [column for column in features.columns if column not in self.base_features]
self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)
 feature_names = list(features.columns)
 features = np.array(features)
 labels = np.array(self.labels ).reshape(( -1,))
feature_importances = feature_importances.sort_values('importance', ascending = False ).reset_index(drop = True)
 feature_importances['normalized_importance'] = feature_importances['importance'] / feature_importances['importance'].sum()
feature_importances['cumulative_importance'] = np.cumsum(feature_importances['normalized_importance'])
 record_zero_importance = feature_importances[feature_importances['importance'] == 0.0]
to_drop = list(record_zero_importance['feature'])
 self.feature_importances = feature_importances
self.record_zero_importance = record_zero_importance
self.ops['zero_importance'] = to_drop
print('
%d features with zero importance after one-hot encoding.
 ' % len(self.ops['zero_importance']))
def identify_low_importance(self, cumulative_importance):

 self.cumulative_importance = cumulative_importance
if self.feature_importances is None:
raise NotImplementedError()
self.feature_importances = self.feature_importances.sort_values('cumulative_importance')
 record_low_importance = self.feature_importances[self.feature_importances['cumulative_importance'] > cumulative_importance]
to_drop = list(record_low_importance['feature'])
 self.record_low_importance = record_low_importance
self.ops['low_importance'] = to_drop
print('%d features required for cumulative importance of %0.2f after one hot encoding.' %(len(self.feature_importances)-
len(self.record_low_importance), self.cumulative_importance))
print('%d features do not contribute to cumulative importance of %0.2f.
 ' %(len(self.ops['low_importance']),
self.cumulative_importance))
def identify_all(self, selection_params):

 for param in ['missing_threshold', 'correlation_threshold', 'eval_metric', 'task', 'cumulative_importance']:
if param not in selection_params.keys() :
raise ValueError('%s is a required parameter for this method.' % param)
 self.identify_missing(selection_params['missing_threshold'])
 self.identify_single_unique()
self.identify_collinear(selection_params['correlation_threshold'])
 self.identify_zero_importance(task = selection_params['task'], eval_metric = selection_params['eval_metric'])
 self.identify_low_importance(selection_params['cumulative_importance'])
 self.all_identified = set(list(chain(*list(self.ops.values()))))
self.n_identified = len(self.all_identified)
 print('%d total features out of %d identified for removal after one-hot encoding.
 ' %(self.n_identified,
self.data_all.shape[1]))
def check_removal(self, keep_one_hot=True):

 self.all_identified = set(list(chain(*list(self.ops.values()))))
print('Total of %d features identified for removal' % len(self.all_identified))
if not keep_one_hot:
if self.one_hot_features is None:
print('Data has not been one-hot encoded')
 else:
one_hot_to_remove = [x for x in self.one_hot_features if x not in self.all_identified]
print('%d additional one-hot features can be removed' % len(one_hot_to_remove))
return list(self.all_identified)
 def remove(self, methods, keep_one_hot = True):

 features_to_drop = []
if methods == 'all':
data = self.data_all
print('{} methods have been run
'.format(list(self.ops.keys())))
 features_to_drop = set(list(chain(*list(self.ops.values()))))
else:
if 'zero_importance' in methods or 'low_importance' in methods or self.one_hot_correlated:
data = self.data_all
else:
data = self.data
for method in methods:
if method not in self.ops.keys() :
raise NotImplementedError('%s method has not been run' % method)
 else:
features_to_drop.append(self.ops[method])
 features_to_drop = set(list(chain(*features_to_drop)))
 features_to_drop = list(features_to_drop)
 if not keep_one_hot:
if self.one_hot_features is None:
print('Data has not been one-hot encoded')
 else:
features_to_drop = list(set(features_to_drop)| set(self.one_hot_features))
data = data.drop(columns = features_to_drop)
 self.removed_features = features_to_drop
if not keep_one_hot:
	print('Removed %d features including one-hot features.' % len(features_to_drop))
else:
	print('Removed %d features.' % len(features_to_drop))
return data
def plot_missing(self):

 if self.record_missing is None:
raise NotImplementedError("Missing values have not been calculated.Run `identify_missing`")
 self.reset_plot()
plt.style.use('seaborn-white')
 plt.figure(figsize =(7, 5))
plt.hist(self.missing_stats['missing_fraction'], bins = np.linspace(0, 1, 11), edgecolor = 'k', color = 'red', linewidth = 1.5)
 plt.xticks(np.linspace(0, 1, 11)) ;
plt.xlabel('Missing Fraction', size = 14); plt.ylabel('Count of Features', size = 14);
plt.title("Fraction of Missing Values Histogram", size = 16);
def plot_unique(self):

 if self.record_single_unique is None:
raise NotImplementedError('Unique values have not been calculated.Run `identify_single_unique`')
 self.reset_plot()
self.unique_stats.plot.hist(edgecolor = 'k', figsize =(7, 5))
plt.ylabel('Frequency', size = 14); plt.xlabel('Unique Values', size = 14);
plt.title('Number of Unique Values Histogram', size = 16);
def plot_collinear(self, plot_all = False):

 if self.record_collinear is None:
raise NotImplementedError('Collinear features have not been idenfitied.Run `identify_collinear`.')
 if plot_all:
	corr_matrix_plot = self.corr_matrix
	title = 'All Correlations'
else:
	
	
	 corr_matrix_plot = self.corr_matrix.loc[list(set(self.record_collinear['corr_feature'])) ,
	 list(set(self.record_collinear['drop_feature'])) ]
	 title = "Correlations Above Threshold"
f, ax = plt.subplots(figsize=(10, 8))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
 sns.heatmap(corr_matrix_plot, cmap=cmap, center=0,
linewidths=.25, cbar_kws={"shrink": 0.6})
 ax.set_yticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[0])) ])
 ax.set_yticklabels(list(corr_matrix_plot.index), size = int(160 / corr_matrix_plot.shape[0])) ;
ax.set_xticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[1])) ])
 ax.set_xticklabels(list(corr_matrix_plot.columns), size = int(160 / corr_matrix_plot.shape[1])) ;
plt.title(title, size = 14)
 def plot_feature_importances(self, plot_n = 15, threshold = None):

 if self.record_zero_importance is None:
raise NotImplementedError('Feature importances have not been determined.Run `idenfity_zero_importance`')
 if plot_n > self.feature_importances.shape[0]:
plot_n = self.feature_importances.shape[0] - 1
self.reset_plot()
plt.figure(figsize =(10, 6))
ax = plt.subplot()
ax.barh(list(reversed(list(self.feature_importances.index[:plot_n]))),
self.feature_importances['normalized_importance'][:plot_n],
align = 'center', edgecolor = 'k')
 ax.set_yticks(list(reversed(list(self.feature_importances.index[:plot_n]))))
ax.set_yticklabels(self.feature_importances['feature'][:plot_n], size = 12)
 plt.xlabel('Normalized Importance', size = 16); plt.title('Feature Importances', size = 18)
 plt.show()
plt.figure(figsize =(6, 4))
plt.plot(list(range(1, len(self.feature_importances)+ 1)) , self.feature_importances['cumulative_importance'], 'r-')
 plt.xlabel('Number of Features', size = 14); plt.ylabel('Cumulative Importance', size = 14);
plt.title('Cumulative Feature Importance', size = 16);
if threshold:
importance_index = np.min(np.where(self.feature_importances['cumulative_importance'] > threshold))
plt.vlines(x = importance_index + 1, ymin = 0, ymax = 1, linestyles='--', colors = 'blue')
 plt.show() ;
print('%d features required for %0.2f of cumulative importance' %(importance_index + 1, threshold))
def reset_plot(self):
plt.rcParams = plt.rcParamsDefault
tr = pd.read_csv('.. /input/X_train.csv')
 te = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv')
 data = pd.read_csv('.. /input/X_train.csv' )<count_values>
fix_cat_params = {
'task_type':"GPU",
'loss_function':'MAE',
'eval_metric':'MAE',
'verbose': False
}
cat_params = {**fix_cat_params, **cat_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_X,
train_target=train_y,
test_data=test_X,
model_fn=run_cat,
params=cat_params,
scoring=mean_absolute_error,
label="cat",
feature_imp=True,
n_folds=8
)
len(tr.measurement_number.value_counts() )<count_values>
submission.to_csv('submission.csv',index=True)
 fs = FeatureSelector(data = train_X, labels = train_y)
 feature_importance = feature_importance_df[["feature", "importance"]].groupby("feature" ).mean()
feature_importance['feature'] = feature_importance.index
print(len(tr.series_id.value_counts()))
print(128*3810,tr.shape )<count_values>
one_hot_features = fs.one_hot_features
base_features = fs.base_features
print('There are %d original features' % len(base_features))
print('There are %d one-hot features' % len(one_hot_features))
len(te.measurement_number.value_counts() )<count_values>
all_to_remove = fs.check_removal()
len(te.series_id.value_counts() )<count_values>
train_removed_all = fs.remove(methods = 'all', keep_one_hot=False )
len(target.series_id.value_counts() )<count_values>
def LGB_bayesian_sec(**lgb_bayes_params):
lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'])
 lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])
 lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])
 lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])
 lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])
 lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])
 lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])
 lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])
 fix_lgb_params = {
'objective':'regression',
"boosting": "gbdt",
"metric": 'mae',
}
lgb_params = {**fix_lgb_params, **lgb_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_df,
train_target=train_y,
test_data=test_df,
model_fn=run_lgb,
params=lgb_params,
scoring=mean_absolute_error,
label="lgb",
feature_imp=True,
n_folds=8 
)
 return -np.mean(scores )
target.surface.value_counts()<count_missing_values>
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_df,
train_target=train_y,
test_data=test_df,
model_fn=run_cat,
params=cat_params,
scoring=mean_absolute_error,
label="cat",
feature_imp=True,
n_folds=8
)
<count_unique_values><EOS> 
submission.time_to_failure = predictions
submission.to_csv('clean_submission.csv',index=True )