Kaggle competition: Jane Street Market Prediction
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
from tensorflow import keras
import pandas as pd
import json
import numpy as np 
<load_from_csv>
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import kerastuner as kt
def read_dataset(path):
return json.load(open(path))
train = read_dataset('.. /input/train.json')
 test = read_dataset('.. /input/test.json' )<string_transform>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
def generate_text(data):
text_data = [" ".join(doc['ingredients'] ).lower() for doc in data]
return text_data<define_variables>
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=32, epochs=1,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )
train_text = generate_text(train)
 test_text = generate_text(test)
 target = [doc['cuisine'] for doc in train] 
<feature_engineering>
TRAINING = False
USE_FINETUNE = False
FOLDS = 4
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
tfidf = TfidfVectorizer(binary=True)
 def tfidf_features(txt, flag):
if flag == "train":
x = tfidf.fit_transform(txt)
 else:
x = tfidf.transform(txt)
 x = x.astype('float16')
 return x
X = tfidf_features(train_text, flag="train")
 X_test = tfidf_features(test_text, flag="test" )<categorify>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
lb = LabelEncoder()
y = lb.fit_transform(target)
 y = keras.utils.to_categorical(y )<choose_model_class>
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.0,0.5))(x)
 for i in range(hp.Int('num_layers',1,3)) :
x = Dense(hp.Int('num_units_{i}',64,256))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.001)) ,loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
model = keras.Sequential()
model.add(keras.layers.Dense(1000, kernel_initializer=keras.initializers.he_normal(seed=1), activation='relu', input_dim=3010))
model.add(keras.layers.Dropout(0.81))
model.add(keras.layers.Dense(1000, kernel_initializer=keras.initializers.he_normal(seed=2), activation='relu'))
model.add(keras.layers.Dropout(0.81))
model.add(keras.layers.Dense(20, kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=0.05, seed=4), activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/fork-of-notebookd9779f8f26/encoder.hdf5')
 encoder.trainable = False
history = model.fit(X, y, epochs=20, batch_size=512, validation_split=0.1)
 model.save_weights("model.h5")
 print("Saved model to disk")
<predict_on_test>
if not TRAINING:
f = np.median
models = models[-2:]
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )