Kaggle competition: Don't Overfit! II
def create_resnet(n_features, n_labels, encoder, label_smoothing = 0.0005):
input_1 = tf.keras.layers.Input(shape =(n_features,))
input_2 = encoder(input_1)
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, activation="elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(512, "elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1), name='l2_norm'),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, ], outputs = output)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer=opt,
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=['AUC'])
 return model<load_pretrained>
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )
%%time
if NN_NAME == '1dcnn':
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_1dcnn(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-1dcnn-for-feature-extraction-train/model_{SEED}_{fold}.hdf5'))
models.append(model)
 models = [models[-1]]<load_pretrained>
null_cnt = train.isnull().sum().sort_values()
print('null count:', null_cnt[null_cnt > 0] )
%%time
if NN_NAME == 'resnet':
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_resnet(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-resnet-with-autoencoder-train/model_{SEED}_{fold}.hdf5'))
models.append(model)
 models = [models[-1]]<load_pretrained>
_=
X = train.drop(['id','target'], axis=1)
 y = train['target']
tr_ids = train['id']
te_ids = test['id']
rfc = RandomForestClassifier(n_estimators=500, class_weight='balanced', max_depth=5, random_state=42)
 selector = RFE(rfc, n_features_to_select=200)
 selector.fit(X, y)
 selected = train.drop(['id','target'], axis=1 ).columns[selector.get_support() ]
train = train[selected]
train['id'] = tr_ids
train['target'] = y
test = test[selected]
test['id'] = te_ids
%%time
if NN_NAME == 'mlp':
model = tf.keras.models.load_model('.. /input/jane-street-with-keras-nn-overfit/model.h5')
 models = [model]<predict_on_test>
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()
f = np.median
th = 0.500
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
batch = treelite_runtime.Batch.from_npy2d(x_tt)
 xgb_pred = predictor.predict(batch)
 if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(0.9*pred + 0.1*xgb_pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices<import_modules>
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<load_from_csv>
cols = [col for col in all_data.columns if col not in ['id','target']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
print(i)
 all_data[i] = pd.factorize(all_data[i])[0]
TRAINING = False
USE_FINETUNE = False
FOLDS = 4
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
numcols = all_data.drop(['id','target'],axis=1 ).select_dtypes(include='number' ).columns.values
scaler = preprocessing.RobustScaler()
all_data.loc[:,numcols] = scaler.fit_transform(all_data[numcols] )
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.005),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
X_train = all_data[all_data['target'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['target'].isnull() ].drop(['target'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
X_train.drop(['id'], axis=1, inplace=True)
 X_test_ID = X_test.pop('id')
 Y_train = X_train.pop('target')
 print(X_train.shape, X_test.shape )
def create_model(input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(0.13 )(x)
 hidden_units = [384, 896, 896, 394]
for idx, hidden_unit in enumerate(hidden_units):
x = Dense(hidden_unit )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.relu )(x)
 x = Dropout(0.25 )(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(0.0005),loss=BinaryCrossentropy(label_smoothing=0.05),metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<categorify>
X_train = pd.concat([X_train, pd.get_dummies(X_train_km, prefix='_km')], axis=1)
 X_test = pd.concat([X_test, pd.get_dummies(X_test_km, prefix='_km')], axis=1 )
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/janestreetmodels/encoder.hdf5')
 encoder.trainable = False<train_model>
FOLDS = 4
SEED = 42
if TRAINING:
gkf = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))
for fold,(train_indices, test_indices)in enumerate(splits):
model = create_model(130, 5, encoder)
 X_train, X_test = X[train_indices], X[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=100,batch_size=4096,callbacks=[EarlyStopping('val_auc',mode='max',patience=10,restore_best_weights=True)])
 model.save_weights(f'./model_{SEED}_{fold}.hdf5')
 model.compile(Adam(0.00001),loss='binary_crossentropy')
 model.fit(X_test,y_test,epochs=3,batch_size=4096)
 model.save_weights(f'./model_{SEED}_{fold}_finetune.hdf5')
 else:
models = []
for f in range(FOLDS):
model = create_model(130, 5, encoder)
 if USE_FINETUNE:
model.load_weights(f'.. /input/janestreetmodels/model_{SEED}_{f}_finetune.hdf5')
 else:
model.load_weights(f'.. /input/janestreetmodels/model_{SEED}_{f}.hdf5')
 models.append(model)
<feature_engineering>
from sklearn.linear_model import LogisticRegression, Lasso
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.feature_selection import RFE, RFECV
<import_modules><EOS> 
submission = pd.DataFrame({
'id': X_test_ID,
'target': sub_preds
})
 submission.to_csv("submission.csv", index=False )