Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<define_variables>
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 np.random.seed(1234)
 print(os.listdir(".. /input"))
training= [df1[features],df2[features],df3[features]]
targets = [df1["action"],df2["action"],df3["action"]]<split>
class PipelineWithCoef(Pipeline):

 def __init__(self, steps, memory=None):
super(PipelineWithCoef, self ).__init__(steps, memory)
 def fit(self, X, y=None, **fit_params):

 super(PipelineWithCoef, self ).fit(X, y, **fit_params)
 self.coef_ = self.steps[-1][1].coef_
return self
datas = []
for i in range(3):
xtr,xval,ytr,yval = train_test_split(training[i].values,targets[i].values,test_size=0.1,\
stratify=targets[i].values)
 datas.append(((xtr,ytr),(xval,yval)) )<drop_column>
target_column = 'target'
converters = {target_column: lambda v: int(float(v)) }
train_df = pd.read_csv('.. /input/train.csv', index_col=0, converters=converters)
 feature_columns = train_df.columns.drop(target_column)
 print(train_df.shape)
 train_df.head()
train_df.target.value_counts()
del(df1)
 del(df2)
 del(df3)
 del(abundant_class)
 del(rare_class)
 del(features_with_missing_values)
 del(correlation_table)
 del(missing_table )<set_options>
( train_df.isnull().sum() > 0 ).any()
gc.collect()<prepare_x_and_y>
target_correlations = corr[target_column].drop(target_column, axis=0)
 top_target_correlations = target_correlations.abs().sort_values(ascending=False)[:50]
top_corr_feature_columns = list(top_target_correlations.index)
 top_target_correlations[:20]
params={"num_leaves":300,
"max_bin":450,
"feature_fraction":0.52,
"bagging_fraction":0.52,
"objective":"binary",
"learning_rate":0.05,
"boosting_type":"gbdt",
"metric":"auc"
}
models = []
for i in range(3):
xtr = datas[i][0][0]
ytr = datas[i][0][1]
xval = datas[i][1][0]
yval = datas[i][1][1]
d_train = lgbm.Dataset(xtr,label=ytr)
 d_eval = lgbm.Dataset(xval,label=yval,reference=d_train)
 clf = lgbm.train(params,d_train,valid_sets=[d_train,d_eval],num_boost_round=1500,\
early_stopping_rounds=50,verbose_eval=50)
 clf.save_model("weights_{}".format(i))
models.append(clf )<set_options>
std = StandardScaler()
X_scaled = std.fit_transform(train_df[feature_columns])
 X_scaled = pd.DataFrame(X_scaled, columns=feature_columns )
env = janestreet.make_env()<define_variables>
lr = LogisticRegression(solver='liblinear', random_state=42)
 param_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2','l1'],
'C' : [0.001, 0.01, 0.1, 1, 10, 100, 1000]}
grid = GridSearchCV(estimator=lr, param_grid=param_grid, iid=False, cv=3, scoring='roc_auc', n_jobs=-1)
 grid.fit(X_scaled[feature_columns], train_df[target_column])
 print("Grid best score was: {}".format(grid.best_score_))
lr = LogisticRegression(solver='liblinear', **grid.best_params_)
 lr.fit(X_scaled[feature_columns], train_df[target_column])
 rfe_lr = RFE(lr, 25, step=1)
 rfe_lr.fit(X_scaled, train_df[target_column] )
th = 0.5000<predict_on_test>
test_df = pd.read_csv('.. /input/test.csv', index_col=0)
 X_test_scaled = std.transform(test_df[feature_columns])
 X_test_scaled = pd.DataFrame(X_test_scaled, columns=feature_columns)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = rfe_lr.predict_proba(X_test_scaled)
 submission.to_csv('rfe_lr_submission.csv', index=False )
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df["weight"].item() > 0 :
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* imputer
pred = np.mean(np.stack([model.predict(x_tt)for model in models]),axis=0 ).T
pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else :
pred_df.action = 0
env.predict(pred_df )<prepare_x_and_y>
def test_model(model, parameter_grid, train_feature_values, train_target_values, retrain_with_top_features=True, importance_threshold=0.0005):

 metric = metrics.make_scorer(metrics.roc_auc_score, greater_is_better=True, needs_proba=True)
 def cv_train_model(model, parameter_grid, train_feature_values, train_target_values):

 folds = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=1234)
 folds_iterable = folds.split(train_feature_values, train_target_values)
 grid_search = GridSearchCV(model, param_grid=parameter_grid, scoring=metric, iid=False, refit=True, cv=folds_iterable, return_train_score=True, n_jobs=-1)
 grid_search_model = grid_search.fit(train_feature_values, train_target_values)
 cv_results = pd.DataFrame.from_dict(grid_search_model.cv_results_)
 cv_results.set_index('rank_test_score', drop=True, inplace=True)
 cv_results.sort_index(inplace=True)
 cv_result_columns = [column for column in cv_results.columns if column.startswith('param_')] + ['mean_test_score', 'std_test_score', 'mean_train_score', 'std_train_score']
display(cv_results[cv_result_columns].head(5))
return grid_search_model
print("Training first model...")
 grid_search_model = cv_train_model(model, parameter_grid, train_feature_values, train_target_values)
 print("Checking feature importances...")
 perm = eli5.sklearn.PermutationImportance(grid_search_model, scoring=metric, n_iter=10, random_state=1234 ).fit(train_feature_values, train_target_values)
 display(eli5.show_weights(perm, top=25))
features_importance_df = eli5.formatters.as_dataframe.explain_weights_df(perm)
 top_feature_columns = [train_feature_values.columns[int(feature_column[1:])]
for feature_column
in features_importance_df[features_importance_df.weight > importance_threshold].feature]
if len(top_feature_columns)== 0:
print("No top feature columns found.Exiting function.")
 return grid_search_model, list(train_feature_values.columns)
 top_feature_columns = top_feature_columns[:50]
if retrain_with_top_features and len(top_feature_columns)< len(train_feature_values.columns):
print("Retraining with top {} features.".format(len(top_feature_columns)))
 grid_search_model = cv_train_model(model, parameter_grid, train_feature_values[top_feature_columns], train_target_values)
 elif retrain_with_top_features:
print("All features are important.Therefore not retraining.")
 top_feature_columns = list(train_feature_values.columns)
 else:
print("Not retraining with top features.")
 top_feature_columns = list(train_feature_values.columns)
 return grid_search_model, top_feature_columns
data = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 data=data[data.weight!=0]
NAN_VALUE = -9999
features = [c for c in data.columns if 'feature' in c]
target = 'resp'
X = data.loc[:, features].fillna(NAN_VALUE)
 Y =(data.loc[:, target] > 0 ).astype(int)
 del data
gc.collect()
model = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.7,
missing=NAN_VALUE,
random_state=2020,
tree_method='gpu_hist',
nthread=multiprocessing.cpu_count() 
)
 model.fit(X, Y)
 print('Finished training model')
 del X, Y
gc.collect()
env = janestreet.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
X_test = test_df.loc[:, features]
X_test = X_test.fillna(NAN_VALUE)
 sample_prediction_df.action = model.predict(X_test)
 env.predict(sample_prediction_df )<install_modules>
all_scores = dict()
!pip --quiet install.. /input/treelite/treelite-0.93-py3-none-manylinux2010_x86_64.whl<install_modules>
sc_lr = Pipeline(steps=[('sc', StandardScaler(with_mean=True, with_std=True)) , 
('logistic_regression', LogisticRegression(solver='liblinear', max_iter=100, random_state=42)) ])
 param_grid = {'logistic_regression__class_weight': ['balanced', None],
'logistic_regression__penalty': ['l1', 'l2'],
'logistic_regression__C': [0.01, 0.1, 0.5, 1., 1.5, 10.]}
sc_lr_model, sc_lr_feature_columns = test_model(sc_lr, param_grid, train_df[feature_columns], train_df[target_column])
 all_scores['sc_lr'] = sc_lr_model.best_score_
!pip --quiet install.. /input/treelite/treelite_runtime-0.93-py3-none-manylinux2010_x86_64.whl<import_modules>
sc_pca_lr = Pipeline(steps=[('sc', StandardScaler(with_mean=True, with_std=True)) , 
('pca', PCA()), 
('logistic_regression', LogisticRegression(solver='liblinear', max_iter=1000)) ])
 param_grid = {'pca__n_components': [5, 10, 15],
'logistic_regression__class_weight': ['balanced', None],
'logistic_regression__penalty': ['l1', 'l2'],
'logistic_regression__C': [0.1, 1., 10.]}
sc_pca_lr_model, sc_pca_lr_feature_columns = test_model(sc_pca_lr, param_grid, train_df[feature_columns], train_df[target_column], importance_threshold=0.001)
 all_scores['sc_pca_lr'] = sc_pca_lr_model.best_score_
import numpy as np
import pandas as pd
import os
import treelite
import treelite_runtime
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
import pathlib
from tqdm import tqdm
from random import choices
import operator
import xgboost as xgb 
<set_options>
sc_lda_lr = Pipeline(steps=[('sc', StandardScaler(with_mean=True, with_std=True)) , 
('lda', LinearDiscriminantAnalysis()), 
('logistic_regression', LogisticRegression(solver='liblinear', max_iter=1000)) ])
 param_grid = {'lda__n_components': [5, 10, 15],
'logistic_regression__class_weight': ['balanced', None],
'logistic_regression__penalty': ['l1', 'l2'],
'logistic_regression__C': [0.1, 1., 10.]}
sc_lda_lr_model, sc_lda_lr_feature_columns = test_model(sc_lda_lr, param_grid, train_df[feature_columns], train_df[target_column])
 all_scores['sc_lda_lr'] = sc_lda_lr_model.best_score_
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
MIXED_PRECISION = False
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )<define_variables>
svc = Pipeline(steps=[('svc', SVC(kernel='linear', gamma='auto', probability=True)) ])
 param_grid = {'svc__C': [0.01, 0.1, 0.5, 1., 1.5, 10., 100.],
'svc__class_weight': ['balanced', None]}
svc_model, svc_feature_columns = test_model(svc, param_grid, train_df[top_corr_feature_columns], train_df[target_column])
 all_scores['svc'] = svc_model.best_score_
SEED = 2021
START_DATE = 86
FOLDS = 5
NN_NAME = 'mlp'<load_from_csv>
def test_model_with_rfe(best_model, train_feature_values, train_target_values, min_nr_features=10, max_nr_features=40):

 deepcopied_best_model = deepcopy(best_model.best_estimator_)
 nr_feaures_to_select_range = list(range(min_nr_features, max_nr_features + 1))
all_scores = []
all_estimators = []
for n_features_to_select in tqdm(nr_feaures_to_select_range):
best_pipeline = PipelineWithCoef(steps=deepcopied_best_model.steps)
 best_pipeline.fit(train_feature_values, train_target_values)
 estimator = RFE(best_pipeline, n_features_to_select=n_features_to_select)
 estimator.fit(train_feature_values, train_target_values)
 folds = RepeatedStratifiedKFold(n_splits=5, n_repeats=1, random_state=1234)
 folds_iterable = folds.split(train_feature_values, train_target_values)
 scores = cross_val_score(estimator, train_feature_values, train_target_values, scoring='roc_auc', cv=folds_iterable)
 all_scores.append(scores.mean())
 all_estimators.append(estimator)
 def plot_roc_auc_scores(nr_features_list, scores_list):

 fig, ax = plt.subplots(figsize=(20, 4))
ax.plot(nr_features_list, scores_list)
 for s in ax.spines:
ax.spines[s].set_visible(False)
 plt.xticks(fontsize=12)
 plt.yticks(fontsize=12)
 ax.set_title('ROC-AUC Score', fontsize=16)
 ax.set_xlabel('Nr Features Selected', fontsize=14);
plot_roc_auc_scores(nr_feaures_to_select_range, all_scores)
 top_index = np.argmax(all_scores)
 print("Number of features selected for best score: {}".format(nr_feaures_to_select_range[top_index]))
best_estimator = all_estimators[top_index]
return best_estimator
%%time
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query(f'date >= {START_DATE}' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<predict_on_test>
rfe_sc_lr_model = test_model_with_rfe(sc_lr_model, train_df[feature_columns], train_df[target_column], max_nr_features=30 )
predictor = treelite_runtime.Predictor('.. /input/janestreet-faster-inference-by-xgb-with-treelite/mymodel.so', verbose=True )<categorify>
test_df = pd.read_csv('.. /input/test.csv', index_col=0)
 print(test_df.shape)
 test_df.head()
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(noise )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = tf.keras.models.Model(inputs=i,outputs=encoded)
 autoencoder = tf.keras.models.Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
def create_submission_file(model, test_data, prefix_output_file):
submission_df = pd.DataFrame(data={'id': test_data.index,
'target': model.predict_proba(test_data)[:,1]})
 submission_df.to_csv(prefix_output_file + '_submission.csv', index=False )
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/janestreet-1dcnn-for-feature-extraction-train/encoder.hdf5')
 encoder.trainable = False<categorify>
create_submission_file(sc_lr_model, test_df[sc_lr_feature_columns], 'sc_lr')
 create_submission_file(svc_model, test_df[svc_feature_columns], 'svc' )
def create_1dcnn(input_dim, output_dim, encoder):
inputs = tf.keras.layers.Input(input_dim)
 x = encoder(inputs)
 x = tf.keras.layers.Concatenate()([x,inputs])
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dense(4096, activation='relu' )(x)
 x = tf.keras.layers.Reshape(( 256, 16))(x)
 x = tf.keras.layers.Conv1D(filters=16,
kernel_size=7,
strides=1,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 x = tf.keras.layers.Flatten()(x)
 for i in range(2):
x = tf.keras.layers.Dense(256 //(2 ** i), activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.GaussianNoise(0.01 )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs,outputs=x)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-02)
 model.compile(optimizer=opt,
loss=loss,
metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<categorify>
create_submission_file(rfe_sc_lr_model, test_df[feature_columns], 'rfe_sc_lr' )