Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<drop_column>
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pandas as pd
import pathlib
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
train['num_ingredients'] = train['ingredients'].apply(len)
 train = train[train['num_ingredients'] > 1]
train.drop('num_ingredients', axis=1, inplace=True )<categorify>
def case_normalization(ingredients):
ingredients = ' '.join(ingredients ).lower()
ingredients = ingredients.replace('-','')
 return ingredients<feature_engineering>
CV_STRATEGY = 'StratifiedGroupKFold'
SEED = 2021
START_DATE = 86
FOLDS = 5
DEBUG = False
train['ingredients'] = train['ingredients'].apply(case_normalization)
 test['ingredients'] = test['ingredients'].apply(case_normalization)
 train.head()<string_transform>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
def clean_data(ingredients):
lemmatizer = WordNetLemmatizer()
ingredients = re.sub(r'[^\w\s\d]', '', ingredients)
 ingredients = re.sub(r'\d+', '', ingredients)
 tokens = word_tokenize(ingredients)
 words = []
for token in tokens:
word = lemmatizer.lemmatize(token)
 if len(word)> 0:
words.append(word)
 return ' '.join(words)
<feature_engineering>
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query(f'date >= {START_DATE}' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
train['features'] = train['ingredients'].apply(clean_data)
 test['features'] = test['ingredients'].apply(clean_data)
 train.head()<categorify>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(noise )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = tf.keras.models.Model(inputs=i,outputs=encoded)
 autoencoder = tf.keras.models.Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
pipeline = Pipeline([ 
("tfidf", TfidfVectorizer()), 
("functiontransform", FunctionTransformer(lambda x: x.astype('float32'), validate=False))
])
 x_train = pipeline.fit_transform(train['features'].values)
 x_train.sort_indices()
x_test = pipeline.transform(test['features'].values )<categorify>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/janestreet-1dcnn-for-feature-extraction-train/encoder.hdf5')
 encoder.trainable = False
label_encoder = LabelEncoder()
y_train = label_encoder.fit_transform(train['cuisine'].values)
 dict(zip(label_encoder.classes_, label_encoder.transform(label_encoder.classes_)))
<choose_model_class>
def create_model(input_dim, output_dim, encoder):
inputs = tf.keras.layers.Input(input_dim)
 x = encoder(inputs)
 x = tf.keras.layers.Concatenate()([x,inputs])
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dense(4096, activation='relu' )(x)
 x = tf.keras.layers.Reshape(( 256, 16))(x)
 x = tf.keras.layers.Conv1D(filters=16,
kernel_size=7,
strides=1,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 x = tf.keras.layers.Flatten()(x)
 for i in range(2):
x = tf.keras.layers.Dense(256 //(2 ** i), activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.GaussianNoise(0.01 )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs,outputs=x)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-02)
 model.compile(optimizer=opt,
loss=loss,
metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
estimator = SVC(
 C = 50,
kernel = 'rbf',
gamma = 1.5,
random_state=42 
)
 classifier = OneVsRestClassifier(estimator, n_jobs=-1 )<train_model>
%%time
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_model(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-1dcnn-for-feature-extraction-train/model_{SEED}_{fold}.hdf5'))
models.append(model)
classifier.fit(x_train, y_train )<save_to_csv>
f = np.median
th = 0.5
models = models[-2:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )