Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<data_type_conversions>
%matplotlib inline
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query(f'date >= {START_DATE}' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4'] 
<prepare_x_and_y>
Conv1D, Conv2D, Conv3D, \
Conv2DTranspose, \
AveragePooling1D, AveragePooling2D, \
MaxPooling1D, MaxPooling2D, MaxPooling3D, \
GlobalAveragePooling1D, GlobalAveragePooling2D, \
GlobalMaxPooling1D, GlobalMaxPooling2D, GlobalMaxPooling3D, \
LocallyConnected1D, LocallyConnected2D, \
concatenate, Flatten, Average, Activation, \
RepeatVector, Permute, Reshape, Dot, \
multiply, dot, add, \
PReLU, \
Bidirectional, TimeDistributed, \
SpatialDropout1D, \
BatchNormalization
if DEBUG:
train = train.sample(10000, random_state=SEED)
 X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
from PIL import Image
from zipfile import ZipFile
import h5py
import cv2
from tqdm import tqdm
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(noise )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = tf.keras.models.Model(inputs=i,outputs=encoded)
 autoencoder = tf.keras.models.Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
src_dir = '.. /input'
train_csv = pd.read_csv(os.path.join(src_dir, 'train.csv'))
print(train_csv.shape)
 train_csv.head()
def create_model(n_features, n_labels, encoder, label_smoothing = 0.0005):
input_1 = tf.keras.layers.Input(shape =(n_features,))
input_2 = encoder(input_1)
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, activation="elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(512, "elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1), name='l2_norm'),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, ], outputs = output)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer=opt,
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=['AUC'])
 return model<categorify>
y_cat_train0 = to_categorical(y_train0)
 print(y_cat_train0.shape)
 y_cat_train0[:5]
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/janestreet-resnet-with-autoencoder-train/encoder.hdf5')
 encoder.trainable = False<load_pretrained>
test_csv = pd.read_csv(os.path.join(src_dir, 'test.csv'))
print(test_csv.shape)
 test_csv.head()
%%time
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_model(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-resnet-with-autoencoder-train/model_{SEED}_{fold}.hdf5'))
models.append(model )<feature_engineering>
sample_submission_csv = pd.read_csv(os.path.join(src_dir, 'sample_submission.csv'))
print(sample_submission_csv.shape)
 sample_submission_csv.head()
f = np.median
th = 0.501
model = models[-1]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = model(x_tt, training = False ).numpy()
pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
warnings.filterwarnings('ignore' )
import os
import pandas as pd
import missingno as msn
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
import datatable as dt
import seaborn as sns
from numba import njit<init_hyperparams>
from sklearn.linear_model import Lasso
from sklearn.feature_selection import RFECV
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV, validation_curve
from sklearn.metrics import mean_squared_error, mean_absolute_error, roc_auc_score, r2_score, make_scorer
from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances
ccut = 0.975
dropout = 0.2
dateCut = 'date > 85'
nepochs = 140
print("Running prediction with correlation cutoff: ", ccut)
 print('Layer dropouts set to: ', dropout)
 runjane = True<load_from_csv>
rfe_min_features = 12
rfe_step = 15
rfe_cv = 20
sss_n_splits = 20
sss_test_size = 0.35
grid_search_cv = 20
noise_std = 0.01
r2_threshold = 0.185
random_seed = 213
np.random.seed(random_seed )
if runjane :
print('
total_size = 0
start_path = '.. /input/jane-street-market-prediction'
for path, dirs, files in os.walk(start_path):
for f in files:
fp = os.path.join(path, f)
 total_size += os.path.getsize(fp)
 print("Directory size: " + str(round(total_size/ 1000000, 2)) + 'MB')
 train_raw = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 else :
train_raw = pd.read_csv('train85_100k.csv')
 print("Data is loaded" )<train_model>
def scoring_roc_auc(y, y_pred):
try:
return roc_auc_score(y, y_pred)
 except:
return 0.5
print("Reading in data with cutoff: ", dateCut)
 train_raw=train_raw.query(dateCut ).reset_index(drop = True )<count_duplicates>
x = RobustScaler().fit_transform(np.concatenate(( x_train0, x_test), axis=0))
x_train0 = x[:250]
x_test = x[250:]
correl= train_raw[features].corr().abs()
correl[correl == 1] = 0
for i in range(len(correl)) :
for j in range(i):
correl.iat[j,i] = 0
cflist = correl.unstack().sort_values(ascending=False ).drop_duplicates()
cflcut = cflist[cflist > ccut]
cflcutis = cflcut.index
print("List of correlated features with absolute value > ", ccut)
 feat2drop = []
print("Building list of features to drop")
 for i in range(len(cflcutis)) :
print("Index: %s , value: %f " %(cflcutis[i] , cflcut[i]))
if not(cflcutis[i][1] in feat2drop):
feat2drop.append(cflcutis[i][1])
 print("Features to drop: ", len(feat2drop))
print(feat2drop)
 for feat in feat2drop:
features.remove(feat)
 print("Number of remaining features is: ", len(features))<import_modules>
robust_roc_auc = make_scorer(scoring_roc_auc)
 param_grid = {
'alpha' : [0.022, 0.021, 0.02, 0.019, 0.023, 0.024, 0.025, 0.026, 0.027, 0.029, 0.031],
'tol' : [0.0013, 0.0014, 0.001, 0.0015, 0.0011, 0.0012, 0.0016, 0.0017]
}
print("Building model and stetting up training now!")
<categorify>
class RFECV_wr(RFECV):
def __init__(self, alpha=1.0, tol=0.0001,
step=1, min_features_to_select=1, cv='warn',
scoring=None, verbose=0, n_jobs=None):
estimator = Lasso(alpha=0.031, tol=0.001,
random_state=random_seed, selection='random')
 super().__init__(estimator, step=step,
min_features_to_select=min_features_to_select,
cv=cv, scoring=scoring, verbose=verbose, n_jobs=n_jobs)
 def set_params(self, **params):
if 'alpha' in params:
self.estimator.set_params(alpha=params['alpha'])
 if 'tol' in params:
self.estimator.set_params(tol=params['tol'])
 return self
feature_selector2 = RFECV_wr(min_features_to_select=rfe_min_features,
scoring=robust_roc_auc,
step=rfe_step,
verbose=0,
cv=rfe_cv,
n_jobs=-1 )
train = train_raw.fillna(train_raw.mean())
 print(train.head())
 print("Null values found in train: ", train.isnull().sum().sum())
 train = reduce_mem_usage(train )<prepare_x_and_y>
feature_selector2.get_params()
train = train[train['weight'] != 0]
if 'resp' in train.columns :
train['action'] = np.where(train['resp'] > 0,1,0)
 else:
train['action'] = np.where(train['weight'] > 0,1,0)
 X = train.loc[:, features]
y = train.loc[:, 'action']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
<categorify>
param_range = np.logspace(-1.7, -1.5, 10)
 param_range
X_nptrain = X_train[:200000].to_numpy()
y_nptrain = y_train[:200000].to_numpy()
print("Shape X_train: ", X_train.shape[1])
 dim_lay1 = X_train.shape[1]
dim_lay2 = dim_lay1 / 1.5
dim_lay3 = dim_lay2 / 2
dim_lay4 = min(10,dim_lay3/2 )<normalization>
scorer = make_scorer(roc_auc_score)
 scorer
cv_splitter = StratifiedShuffleSplit(n_splits=5, test_size=0.35, random_state=0)
 cv_splitter.get_n_splits(x_train0, y_train0 )
normalizer = preprocessing.Normalization()
normalizer.adapt(X_nptrain)
 input_shape = X_train.shape[1:]<choose_model_class>
train_scores, test_scores = validation_curve(
 feature_selector2, x_train0, y_train0,
param_name="alpha", param_range=param_range,
cv=cv_splitter, scoring=scorer, n_jobs=None, verbose=0 )
inputs = keras.Input(shape=input_shape)
 x = tf.keras.layers.BatchNormalization()(inputs)
 x = layers.Dense(dim_lay1, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 x = layers.Dense(dim_lay2, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 x = layers.Dense(dim_lay3, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 x = layers.Dense(dim_lay4, activation="tanh" )(x)
 x = layers.Dropout(dropout )(x)
 outputs = layers.Dense(1, activation="sigmoid" )(x)
 model = keras.Model(inputs, outputs)
 model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()<train_model>
train_scores_mean = np.mean(train_scores, axis=1)
 train_scores_std = np.std(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 test_scores_std = np.std(test_scores, axis=1)
 train_scores_mean
%time history = model.fit(X_train, y_train, epochs=nepochs, batch_size = 4096, validation_split=0.05, verbose=1 )<compute_test_metric>
print("counter | val_mse | val_mae | val_roc | val_r2 | alpha | feature_count ")
 print("-------------------------------------------------------------------------------------")
 importances = np.zeros(( 300,))
predictions = pd.DataFrame()
counter = 0
for train_index, val_index in StratifiedShuffleSplit(n_splits=sss_n_splits, test_size=sss_test_size, random_state=random_seed ).split(x_train0, y_train0):
X, val_X = x_train0[train_index], x_train0[val_index]
y, val_y = y_train0[train_index], y_train0[val_index]
grid_search = GridSearchCV(feature_selector2,
param_grid=param_grid,
verbose=0,
n_jobs=None,
scoring=robust_roc_auc,
cv=grid_search_cv)
 grid_search.fit(X, y)
 val_y_pred = grid_search.best_estimator_.predict(val_X)
 val_mse = mean_squared_error(val_y, val_y_pred)
 val_mae = mean_absolute_error(val_y, val_y_pred)
 val_roc = roc_auc_score(val_y, val_y_pred)
 val_cos = cosine_similarity(val_y.reshape(1, -1), val_y_pred.reshape(1, -1)) [0][0]
val_dst = euclidean_distances(val_y.reshape(1, -1), val_y_pred.reshape(1, -1)) [0][0]
val_r2 = r2_score(val_y, val_y_pred)
 if val_r2 > r2_threshold:
message = '<-- OK'
prediction = grid_search.best_estimator_.predict(x_test)
 predictions = pd.concat([predictions, pd.DataFrame(prediction)], axis=1)
 importances += grid_search.best_estimator_.support_.astype(int)
 else:
message = '<-- skipping'
print("{0:2} | {1:.4f} | {2:.4f} | {3:.4f} | {4:.4f} | {5:.4f} | {6:3} {7} ".format(
 counter,
val_mse,
val_mae,
val_roc,
val_r2,
grid_search.best_estimator_.estimator_.get_params() ['alpha'],
grid_search.best_estimator_.n_features_,
message))
counter += 1
score = model.evaluate(X_test, y_test)
 print("Accuracy Score on X_test: "+str(round(score[1],4)) )<define_variables>
grid_search.best_estimator_.estimator_.get_params()
if runjane :
janestreet.make_env.__called__ = False
env = janestreet.make_env()
iter_test = env.iter_test()<predict_on_test>
mean_pred = pd.DataFrame(predictions.mean(axis=1))
mean_pred.index += 250
mean_pred.columns = ['target']
mean_pred.to_csv('submission.csv', index_label='id', index=True)
 mean_pred.head()