Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
%matplotlib inline
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed )<normalization>
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
class NeutralizeTransform:
def __init__(self,proportion=1.0):
self.proportion = proportion
def fit(self,X,y):
self.lms = []
self.mean_exposure = np.mean(y,axis=0)
 self.y_shape = y.shape[-1]
for x in X.T:
scores = x.reshape(( -1,1))
exposures = y
exposures = np.hstack(( exposures, np.array([np.mean(scores)] * len(exposures)).reshape(-1, 1)))
 transform = np.linalg.lstsq(exposures, scores, rcond=None)[0]
self.lms.append(transform)
 def transform(self,X,y=None):
out = []
for i,transform in enumerate(self.lms):
x = X[:,i]
scores = x.reshape(( -1,1))
exposures = np.repeat(self.mean_exposure,len(x),axis=0 ).reshape(( -1,self.y_shape))
exposures = np.concatenate([exposures,np.array([np.mean(scores)] * len(exposures)).reshape(( -1,1)) ],axis=1)
 correction = self.proportion * exposures.dot(transform)
 out.append(x - correction.ravel())
 return np.asarray(out ).T
def fit_transform(self,X,y):
self.fit(X,y)
 return self.transform(X,y)
<define_variables>
mnist_train.isna().any().any()
TRAINING = False<categorify>
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0
%%time
if TRAINING:
mask = train[features].isna()
train.fillna(0,inplace=True)
 for feature in features:
nt = NeutralizeTransform(proportion=0.25)
 train[feature] = nt.fit_transform(train[feature].values.reshape(( -1,1)) ,
train['resp'].values.reshape(( -1,1)))
 pd.to_pickle(nt,f'NeutralizeTransform_{feature}.pkl')
 train[mask] = np.nan
else:
nts = []
for feature in features:
nt = pd.read_pickle(f'{PATH}/NeutralizeTransform_{feature}.pkl')
 nts.append(nt )<set_options>
standardized_scalar = StandardScaler()
standardized_data = standardized_scalar.fit_transform(mnist_train_data)
 standardized_data.shape
gc.collect()<define_variables>
cov_matrix = np.matmul(standardized_data.T, standardized_data)
 cov_matrix.shape
TRAINING = False<prepare_x_and_y>
lambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))
vectors.shape
X_tr = train.query('date<350')[features].values
y_tr =(train.query('date<350')[resp_cols].values > 0 ).astype(int)
 X_val = train.query('date>400')[features].values
y_val =(train.query('date>400')[resp_cols].values > 0 ).astype(int)
 del train
gc.collect()
if TRAINING:
metric = {}
for seed in [2020,1982]:
set_all_seeds(seed)
 model = create_model(X_tr.shape[-1])
 hist = model.fit(X_tr,y_tr,
validation_data=(X_val,y_val),
epochs=200,
batch_size=8192,
callbacks=[tf.keras.callbacks.EarlyStopping('val_binary_accuracy',mode='max',patience=20,restore_best_weights=True),
tf.keras.callbacks.ReduceLROnPlateau('val_binary_accuracy',mode='max',patience=10,cooldown=5)])
 model.save_weights(f'model_{seed}.tf')
 metric[seed] = max(hist.history['val_binary_accuracy'])
 tf.keras.backend.clear_session()
print(metric)
 else:
models = []
for seed in [2020,1982]:
model = create_model(X_tr.shape[-1])
 model.load_weights(f'{PATH}/model_{seed}.tf')
 model.call = tf.function(model.call, experimental_relax_shapes=True)
 models.append(model)
<feature_engineering>
new_coordinates = np.matmul(vectors, standardized_data.T)
 print(new_coordinates.shape)
 new_coordinates = np.vstack(( new_coordinates, mnist_train_label)).T
if not TRAINING:
f = np.median
env = janestreet.make_env()
th = 0.495
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].values[0] > 0:
test_df['feature_stock_id_sum'] = test_df['feature_41'] + test_df['feature_42'] + test_df['feature_43']
test_df['feature_1_2_cross'] = test_df['feature_1']/(test_df['feature_2']+1e-5)
 x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
for i in range(len(nts)) :
x_tt[:,i] = nts[i].transform(np.expand_dims(x_tt[:,i],0))
p = f(np.mean([model(x_tt,training=False ).numpy() for model in models],axis=0))
pred_df.action = np.where(p > th, 1, 0 ).astype(int)
 else:
pred_df["action"].values[0] = 0
env.predict(pred_df )<import_modules>
df_new = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_new.head()
<load_from_csv>
pca = decomposition.PCA()
pca.n_components = 2
pca_data = pca.fit_transform(standardized_data)
 pca_data.shape
start_time = time.time()
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 print('Load in data successful!')
 print('-----------------------------------------------------------------------')
 time_passed_m = int(( time.time() - start_time)// 60)
 time_passed_s = int(( time.time() - start_time)% 60)
 print(f'Time usage: {time_passed_m}m.{time_passed_s}s.' )<define_variables>
pca_data = np.vstack(( pca_data.T, mnist_train_label)).T
SEED = 42
np.random.seed(SEED)
 TRAINING_PGTS = True
MANUAL_VALIDATING = False
TRAINING = True
print(f'TRAINING_PGTS = {TRAINING_PGTS}
\
MANUAL_VALIDATING = {MANUAL_VALIDATING}
\
TRAINING = {TRAINING}' )<prepare_x_and_y>
df_PCA = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_PCA.head()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() , inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if 'feature' in c]
f_mean = np.mean(train[features[1:]].values, axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
print(X_train.shape, y_train.shape)
 if MANUAL_VALIDATING:
X_train_v = train[train['date'] <= 416]
y_train_v = np.stack([(X_train_v[c] > 0 ).astype('int')for c in resp_cols] ).T
X_train_v = X_train_v.loc[:, features].values
X_test_v = train[train['date'] > 416]
y_test_v = np.stack([(X_test_v[c] > 0 ).astype('int')for c in resp_cols] ).T
X_test_v = X_test_v.loc[:, features].values
print(X_test_v.shape, y_test_v.shape)
 print('Done data preprocessing!' )<define_search_model>
mnist_train_data = np.array(mnist_train_data)
 mnist_train_label = np.array(mnist_train_label )
def create_mlp(num_columns, num_labels, hidden_units,
dropout_rates, label_smoothing, learning_rate):
inp = Input(shape=(num_columns,))
x = BatchNormalization()(inp)
 x = Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = Dense(hidden_units[i] )(x)
 x = BatchNormalization()(x)
 x = Activation(tf.keras.activations.swish )(x)
 x = Dropout(dropout_rates[i+1] )(x)
 x = Dense(num_labels )(x)
 out = Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name='AUC')
)
 return model<import_modules>
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D
from tensorflow.keras.optimizers import Adadelta
from keras.utils.np_utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import LearningRateScheduler
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<prepare_x_and_y>
nclasses = mnist_train_label.max() - mnist_train_label.min() + 1
mnist_train_label = to_categorical(mnist_train_label, num_classes = nclasses)
 print("Shape of ytrain after encoding: ", mnist_train_label.shape )
n_samples = 2000
n_groups = 20
assert n_samples % n_groups == 0
idx = np.linspace(0, n_samples-1, num=n_samples)
 X_train_pgts = np.random.random(size=(n_samples, 5))
y_train_pgts = np.random.choice([0, 1], n_samples)
 groups = np.repeat(np.linspace(0, n_groups-1, num=n_groups), n_samples/n_groups)
 groups.shape<split>
def build_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def compile_model(model, optimizer='adam', loss='categorical_crossentropy'):
model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
 def train_model(model, train, test, epochs, split):
history = model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)
 return history
FOLDS = 5
models = []
if TRAINING_PGTS:
gkf = PurgedGroupTimeSeriesSplit(n_splits=FOLDS, group_gap=20)
 splits = list(gkf.split(y_train, groups=train['date'].values))
for fold,(train_indices, test_indices)in tqdm(enumerate(splits)) :
X_train_pgts, X_test_pgts = X_train.iloc[train_indices, :], X_train.iloc[test_indices, :]
y_train_pgts, y_test_pgts = y_train[train_indices], y_train[test_indices]
tf.keras.backend.clear_session()
model = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 er = EarlyStopping(patience = 8,
restore_best_weights = True,
monitor = 'val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.1,
patience = 8,
verbose = 1,
mode = 'min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath = f'model3-1_{SEED}_{fold}.hdf5',
save_weights_only = True,
verbose = 0,
monitor = 'val_loss',
save_best_only = True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
model.fit(X_train_pgts, y_train_pgts,
validation_data =(X_test_pgts, y_test_pgts),
epochs = epochs[0],
batch_size = batch_size[1],
verbose=2,
callbacks = nn_callbacks)
 models.append(model)
 else:
for fold in tqdm(range(FOLDS)) :
tf.keras.backend.clear_session()
model = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 model.load_weights(f'.. /input/js-model-v1/model4_{SEED}_{fold}.hdf5')
 models.append(model )<choose_model_class>
cnn_model = build_model(( 28, 28, 1))
compile_model(cnn_model, 'adam', 'categorical_crossentropy')
 model_history = train_model(cnn_model, mnist_train_data, mnist_train_label, 80, 0.2 )
if MANUAL_VALIDATING:
er = EarlyStopping(patience = 8,
restore_best_weights = True,
monitor = 'val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.1,
patience = 8,
verbose = 1,
mode = 'min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath = 'js_model_v4_weights.h5',
save_weights_only = True,
verbose = 0,
monitor = 'val_loss',
save_best_only = True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 history = clf.fit(
 X_train_v, y_train_v,
epochs = epochs[0],
batch_size = batch_size,
verbose = 2,
validation_data =(X_test_v, y_test_v),
callbacks = nn_callbacks 
)
 del X_train_v, y_train_v, X_test_v, y_test_v
if TRAINING:
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 clf.fit(X_train, y_train, epochs=epochs[0],
batch_size=batch_size[1], verbose=2, 
)
 clf.save_weights('js_model_v2-1_weights.h5')
 print('Training succeeded!
')
 else:
clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 clf.load_weights('.. /input/js-model-v1/model3-1_1111_9.hdf5')
 print('Loading succeeded!
')
 clf.summary()
models.append(clf )<feature_engineering>
predictions = cnn_model.predict(mnist_test_arr )
class JSMP_Dataset(Dataset):
def __init__(self, file_path, window_size):
self.file_path = file_path
self.window_size = window_size
train = pd.read_csv(file_path)
 train = train.query('date > 85' ).reset_index(drop = True)
 train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
self.features = [c for c in train.columns if "feature" in c]
self.f_mean = np.mean(train[self.features[1:]].values,axis=0)
 self.X_train = train.loc[:, train.columns.str.contains('feature')].values
self.y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
self.X_train = torch.from_numpy(self.X_train ).float()
self.y_train = torch.from_numpy(self.y_train ).float()
del train
gc.collect()
def __len__(self):
return len(self.X_train)- self.window_size
def __getitem__(self, i):
data = self.X_train[i:(i+ self.window_size), :]
label = self.y_train[i + self.window_size - 1]
return data, label<create_dataframe>
predictions_test = []
for i in predictions:
predictions_test.append(np.argmax(i))
window_size = 5
file_path = '/kaggle/input/jane-street-market-prediction/train.csv'
ds = JSMP_Dataset(file_path, window_size )<normalization>
submission = pd.DataFrame({
"ImageId": mnist_test.index+1,
"Label": predictions_test
})
 submission.to_csv('my_submission.csv', index=False )
class Chomp1d(nn.Module):
def __init__(self, chomp_size):
super(Chomp1d, self ).__init__()
self.chomp_size = chomp_size
def forward(self, x):
return x[:, :, :-self.chomp_size].contiguous()
class TemporalBlock(nn.Module):
def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):
super(TemporalBlock, self ).__init__()
self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,
stride=stride, padding=padding, dilation=dilation))
self.chomp1 = Chomp1d(padding)
 self.relu1 = nn.ReLU()
self.dropout1 = nn.Dropout(dropout)
 self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,
stride=stride, padding=padding, dilation=dilation))
self.chomp2 = Chomp1d(padding)
 self.relu2 = nn.ReLU()
self.dropout2 = nn.Dropout(dropout)
 self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1,
self.conv2, self.chomp2, self.relu2, self.dropout2)
 self.downsample = nn.Conv1d(n_inputs, n_outputs, 1)if n_inputs != n_outputs else None
self.relu = nn.ReLU()
self.init_weights()
def init_weights(self):
self.conv1.weight.data.normal_(0, 0.01)
 self.conv2.weight.data.normal_(0, 0.01)
 if self.downsample is not None:
self.downsample.weight.data.normal_(0, 0.01)
 def forward(self, x):
out = self.net(x)
 res = x if self.downsample is None else self.downsample(x)
 return self.relu(out + res)
 class TemporalConvNet(nn.Module):
def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
super(TemporalConvNet, self ).__init__()
layers = []
num_levels = len(num_channels)
 for i in range(num_levels):
dilation_size = 2 ** i
in_channels = num_inputs if i == 0 else num_channels[i-1]
out_channels = num_channels[i]
layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,
padding=(kernel_size-1)* dilation_size, dropout=dropout)]
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x )<choose_model_class>
%matplotlib inline
class TCN(nn.Module):
def __init__(self, input_size, output_size, num_channels, kernel_size, dropout):
super(TCN, self ).__init__()
self.tcn = TemporalConvNet(input_size, num_channels, kernel_size=kernel_size, dropout=dropout)
 self.fc1 = nn.Linear(130 * num_channels[-1], 128)
 self.dropout1 = nn.Dropout(dropout)
 self.batch_norm1 = nn.BatchNorm1d(128)
 self.LeakyReLU1 = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.fc2 = nn.Linear(128, 128)
 self.dropout2 = nn.Dropout(dropout)
 self.batch_norm2 = nn.BatchNorm1d(128)
 self.LeakyReLU2 = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.fc3 = nn.Linear(128, output_size)
 def forward(self, inputs):

 y1 = self.tcn(inputs)
 y1 = torch.flatten(y1, start_dim=1)
 y1 = self.fc1(y1)
 y1 = self.batch_norm1(y1)
 y1 = self.LeakyReLU1(y1)
 y1 = self.dropout1(y1)
 y1 = self.fc2(y1)
 y1 = self.batch_norm2(y1)
 y1 = self.LeakyReLU2(y1)
 y1 = self.dropout2(y1)
 o = self.fc3(y1)
 return torch.sigmoid(o )<find_best_params>
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print('use devise:', device)
 net1 = TCN(input_size=5, output_size=5, num_channels=[16, 8, 4, 2], kernel_size=2, dropout=0.5)
 net2 = TCN(input_size=5, output_size=5, num_channels=[16, 8, 4, 2], kernel_size=2, dropout=0.5)
 net1.load_state_dict(torch.load('/kaggle/input/jsmp-tcn-pytorch/best_accuracy_model.mdl', map_location=torch.device(device)))
 net2.load_state_dict(torch.load('/kaggle/input/jsmp-tcn-pytorch/best_loss_model.mdl', map_location=torch.device(device)))
 net1.eval()
net2.eval()<feature_engineering>
mnist_train.isna().any().any()
th = 0.5
env = janestreet.make_env()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print('use devise:', device)
 for i,(test_df, pred_df)in enumerate(env.iter_test()):
x_tt = test_df.loc[:, ds.features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* ds.f_mean
if i == 0:
x_window = x_tt.copy()
elif i < window_size:
x_window = np.concatenate([x_window, x_tt], axis=0)
 else:
x_window = np.concatenate([x_window[1:, :], x_tt], axis=0)
 if i < window_size - 1:
pred_df.action = 0
else:
if test_df['weight'].item() > 0:
inputs = torch.Tensor(x_window ).unsqueeze(0 ).to(device)
 outputs =(net1(inputs)+ net2(inputs)) / 2
pred =(torch.median(outputs, axis=1 ).values > th ).long()
pred_df.action = pred.item()
else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0
import os
import time
import gc
import random
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
import pathlib
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.model_selection import KFold
from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples
from sklearn.utils.validation import _deprecate_positional_args
import tensorflow as tf
from tensorflow import keras
import tensorflow_addons as tfa
from tensorflow.keras import layers<set_options>
standardized_scalar = StandardScaler()
standardized_data = standardized_scalar.fit_transform(mnist_train_data)
 standardized_data.shape
GPUs = tf.config.experimental.list_physical_devices(device_type='GPU')
 for gpu in GPUs:
tf.config.experimental.set_memory_growth(gpu, True )<import_modules>
cov_matrix = np.matmul(standardized_data.T, standardized_data)
 cov_matrix.shape
print('tensorflow_version_is',tf.__version__ )<define_variables>
lambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))
vectors.shape
SEED=42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(seed=SEED )<load_from_disk>
new_coordinates = np.matmul(vectors, standardized_data.T)
 print(new_coordinates.shape)
 new_coordinates = np.vstack(( new_coordinates, mnist_train_label)).T
%%time
print('Loading data...')
 train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 print('Done!' )<normalization>
df_new = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_new.head()
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)} )<data_type_conversions>
pca = decomposition.PCA()
pca.n_components = 2
pca_data = pca.fit_transform(standardized_data)
 pca_data.shape
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train.fillna(train.mean() ,inplace=True)
 base_features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[base_features[1:]].values,axis=0)
 train['action'] =(train['resp'] > 0 ).astype('int')
 train['action_1'] =(train['resp_1'] > 0 ).astype('int')
 train['action_2'] =(train['resp_2'] > 0 ).astype('int')
 train['action_3'] =(train['resp_3'] > 0 ).astype('int')
 train['action_4'] =(train['resp_4'] > 0 ).astype('int')
 def add_features(train_df):
train_df['feature_cross_41_42_43'] = train_df['feature_41']+train_df['feature_42']+train_df['feature_43']
train_df['feature_cross_1_2'] = train_df['feature_1']/(train_df['feature_2']+2e-5)
 return train_df
train = add_features(train)
 features = [c for c in train.columns if "feature" in c]
target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in target_cols] ).T
X.shape, y.shape<choose_model_class>
pca_data = np.vstack(( pca_data.T, mnist_train_label)).T
def RestNet_(num_columns,
num_labels,
hidden_size,
dropout_rate,
label_smoothing,
learning_rate):
inp = layers.Input(shape=(num_columns,))
x = layers.BatchNormalization()(inp)
 x = layers.Dense(132 )(x)
 x = layers.LeakyReLU()(x)
 x = layers.Dropout(0.315 )(x)
 x1 = layers.Dense(hidden_size*1.2 )(x)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('mish' )(x1)
 x1 = layers.Dropout(dropout_rate )(x1)
 x = layers.concatenate([x, x1], axis=1)
 x2 = layers.Dense(hidden_size*1.1 )(x)
 x2 = layers.BatchNormalization(axis=1 )(x2)
 x2 = layers.Activation('mish' )(x2)
 x2 = layers.Dropout(dropout_rate )(x2)
 x = layers.concatenate([x1, x2], axis=1)
 x3 = layers.Dense(hidden_size*1.0 )(x)
 x3 = layers.BatchNormalization(axis=1 )(x3)
 x3 = layers.Activation('mish' )(x3)
 x3 = layers.Dropout(dropout_rate )(x3)
 x = layers.concatenate([x2, x3], axis=1)
 x4 = layers.Dense(hidden_size*0.9 )(x)
 x4 = layers.BatchNormalization(axis=1 )(x4)
 x4 = layers.Activation('mish' )(x4)
 x4 = layers.Dropout(dropout_rate )(x4)
 x = layers.concatenate([x3, x4], axis=1)
 x5 = layers.Dense(hidden_size*0.8 )(x)
 x5 = layers.BatchNormalization(axis=1 )(x5)
 x5 = layers.LeakyReLU()(x5)
 x5 = layers.Dropout(dropout_rate )(x5)
 x = layers.concatenate([x1, x3, x5], axis=1)
 x = layers.Dense(num_labels )(x)
 out = layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate,weight_decay=1e-5),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC")
)
 return model<choose_model_class>
df_PCA = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_PCA.head()
hidden_units = 256
dropout_rates = 0.3
label_smoothing = 5e-3
learning_rate = 1e-3
model = RestNet_(X.shape[1],
y.shape[1],
hidden_units,
dropout_rates,
label_smoothing,
learning_rate)
 model.summary()<set_options>
mnist_train_data = np.array(mnist_train_data)
 mnist_train_label = np.array(mnist_train_label )
del model
gc.collect()<define_search_model>
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D
from tensorflow.keras.optimizers import Adadelta
from keras.utils.np_utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import LearningRateScheduler
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<define_search_model>
nclasses = mnist_train_label.max() - mnist_train_label.min() + 1
mnist_train_label = to_categorical(mnist_train_label, num_classes = nclasses)
 print("Shape of ytrain after encoding: ", mnist_train_label.shape )
NUM_FOLDS = 7
EPOCHS = 500
BATCH_SIZE = 6500
TRAINING = False
CV = True
if TRAINING:
if CV:
gkf = PurgedGroupTimeSeriesSplit(n_splits = NUM_FOLDS, group_gap=15)
 splits = list(gkf.split(y, groups=train['date'].values))
for fold,(train_indices, test_indices)in enumerate(splits):
keras.backend.clear_session()
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
verbose=1,
factor=0.2,
patience=12, mode='min')
 early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=25)
 checkpoint_callback = keras.callbacks.ModelCheckpoint(
 filepath='logs/BestVal_model_{}.h5'.format(fold),
save_weights_only=True,
monitor='val_AUC',
mode='max',
verbose=1,
save_best_only=True)
 model = RestNet_(X.shape[1],
y.shape[1],
hidden_units,
dropout_rates,
label_smoothing,
learning_rate)
 X_train, X_test = X[train_indices], X[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
model.fit(X_train,
y_train,
validation_data=(X_test,y_test),
epochs=EPOCHS,
batch_size=BATCH_SIZE,
callbacks=[reduce_lr,
early_stop,
checkpoint_callback])
 model.save_weights(f'logs/model_{SEED}_{fold}.h5')
 del model
gc.collect()
else:
keras.backend.clear_session()
model = RestNet_(len(features1),
len(features2),
y.shape[1],
hidden_units,
dropout_rates,
label_smoothing,
learning_rate)
 model.fit(X,
y,
epochs=EPOCHS,
batch_size=BATCH_SIZE)
 model.save_weights(f'.logs/model_{SEED}_NONE.h5')
 else:
SEED1=42
Full_models_42 = []
for fold in range(NUM_FOLDS):
tf.keras.backend.clear_session()
model1 = RestNet_(X.shape[1],
y.shape[1],
hidden_units,
dropout_rates,
label_smoothing,
learning_rate)
 model1.load_weights(pathlib.Path(f'.. /input/drop0315/model_{SEED1}_{fold}.h5'))
Full_models_42.append(model1)
 SEED2=73
Full_models_73 = []
for fold in range(NUM_FOLDS):
tf.keras.backend.clear_session()
model2 = RestNet_(X.shape[1],
y.shape[1],
hidden_units,
dropout_rates,
label_smoothing,
learning_rate)
 model2.load_weights(pathlib.Path(f'.. /input/73cv7/model_{SEED2}_{fold}.h5'))
Full_models_73.append(model2)
 SEED3=2021
Full_models_2021 = []
for fold in range(NUM_FOLDS):
tf.keras.backend.clear_session()
model3 = RestNet_(X.shape[1],
y.shape[1],
hidden_units,
dropout_rates,
label_smoothing,
learning_rate)
 model3.load_weights(pathlib.Path(f'.. /input/2021cv7/model_{SEED3}_{fold}.h5'))
Full_models_2021.append(model3)
<categorify>
def build_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def compile_model(model, optimizer='adam', loss='categorical_crossentropy'):
model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
 def train_model(model, train, test, epochs, split):
history = model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)
 return history
%%time
class LiteModel:
@classmethod
def from_file(cls, model_path):
return LiteModel(tf.lite.Interpreter(model_path=model_path))
@classmethod
def from_keras_model(cls, kmodel):
converter = tf.lite.TFLiteConverter.from_keras_model(kmodel)
 tflite_model = converter.convert()
return LiteModel(tf.lite.Interpreter(model_content=tflite_model))
def __init__(self, interpreter):
self.interpreter = interpreter
self.interpreter.allocate_tensors()
input_det = self.interpreter.get_input_details() [0]
output_det = self.interpreter.get_output_details() [0]
self.input_index = input_det["index"]
self.output_index = output_det["index"]
self.input_shape = input_det["shape"]
self.output_shape = output_det["shape"]
self.input_dtype = input_det["dtype"]
self.output_dtype = output_det["dtype"]
def predict(self, inp):
inp = inp.astype(self.input_dtype)
 count = inp.shape[0]
out = np.zeros(( count, self.output_shape[1]), dtype=self.output_dtype)
 for i in range(count):
self.interpreter.set_tensor(self.input_index, inp[i:i+1])
 self.interpreter.invoke()
out[i] = self.interpreter.get_tensor(self.output_index)[0]
return out
def predict_single(self, inp):

 inp = np.array([inp], dtype=self.input_dtype)
 self.interpreter.set_tensor(self.input_index, inp)
 self.interpreter.invoke()
out = self.interpreter.get_tensor(self.output_index)
 return out[0]
tflite_models_42=[]
for i in range(len(Full_models_42)) :
tflite_model_ = LiteModel.from_keras_model(Full_models_42[i])
 tflite_models_42.append(tflite_model_)
 tflite_models_73=[]
for i in range(len(Full_models_73)) :
tflite_model_ = LiteModel.from_keras_model(Full_models_73[i])
 tflite_models_73.append(tflite_model_)
 tflite_models_2021=[]
for i in range(len(Full_models_2021)) :
tflite_model_ = LiteModel.from_keras_model(Full_models_2021[i])
 tflite_models_2021.append(tflite_model_)
 print(len(tflite_models_42)+ len(tflite_models_73)+ len(tflite_models_2021))<define_search_space>
cnn_model = build_model(( 28, 28, 1))
compile_model(cnn_model, 'adam', 'categorical_crossentropy')
 model_history = train_model(cnn_model, mnist_train_data, mnist_train_label, 80, 0.2 )
f = np.median
th = 0.502
weight_model = [1,1,1,2,2,2]<feature_engineering>
predictions = cnn_model.predict(mnist_test_arr )
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
test_df_ = test_df.loc[:, base_features].values
if np.isnan(test_df_[:, 1:].sum()):
test_df_[:, 1:] = np.nan_to_num(test_df_[:, 1:])+ np.isnan(test_df_[:, 1:])* f_mean
cross_41_42_43 = test_df_[:, 41] + test_df_[:, 42] + test_df_[:, 43]
cross_1_2 = test_df_[:, 1] /(test_df_[:, 2] + 2e-5)
 x_tt = np.concatenate((
test_df_,
np.array(cross_41_42_43 ).reshape(test_df_.shape[0], 1),
np.array(cross_1_2 ).reshape(test_df_.shape[0], 1), 
), axis=1)
 pred_42 = np.average([clf.predict(x_tt)for clf in tflite_models_42], axis=0, weights=np.array(weight_model))
pred_42 = f(pred_42)
 pred_73 = np.average([clf.predict(x_tt)for clf in tflite_models_73], axis=0, weights=np.array(weight_model))
pred_42 = f(pred_73)
 pred_2021 = np.average([clf.predict(x_tt)for clf in tflite_models_2021], axis=0, weights=np.array(weight_model))
pred_2021 = f(pred_2021)
 pred = pred_42*0.3 + pred_73*0.3 + pred_73*0.4
pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
predictions_test = []
for i in predictions:
predictions_test.append(np.argmax(i))
!mkdir cache<import_modules>
submission = pd.DataFrame({
"ImageId": mnist_test.index+1,
"Label": predictions_test
})
 submission.to_csv('my_submission.csv', index=False )
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array] 
<import_modules>
%matplotlib inline
class FinData(Dataset):
def __init__(self, data, target, date, mode='train', transform=None, cache_dir=None, multi=False):
self.data = data
self.target = target
self.mode = mode
self.transform = transform
self.cache_dir = cache_dir
self.date = date
self.multi = multi
def __getitem__(self, index):
if torch.is_tensor(index):
index.to_list()
if self.transform:
return self.transform(self.data.iloc[index].values)
 else:
if type(index)is list:
if self.multi == False:
sample = {
'target': torch.Tensor(self.target.iloc[index].values),
'data': torch.FloatTensor(self.data[index]),
'date': torch.Tensor(self.date.iloc[index].values)
 }
elif self.multi == True:
sample = {
'target': torch.Tensor(self.target[index]),
'data': torch.FloatTensor(self.data[index]),
'date': torch.Tensor(self.date.iloc[index].values)
 }
else:
if self.multi == False:
sample = {
'target': torch.Tensor(self.target.iloc[index]),
'data': torch.FloatTensor(self.data[index]),
'date': torch.Tensor(self.date.iloc[index])
 }
elif self.multi == True:
sample = {
'target': torch.Tensor(self.target[index]),
'data': torch.FloatTensor(self.data[index]),
'date': torch.Tensor(self.date.iloc[index])
 }
return sample
def __len__(self):
return len(self.data)
 def load_data(root_dir, mode, overide=None):
if overide:
data = dt.fread(overide ).to_pandas()
elif mode == 'train':
data = dt.fread(root_dir + 'train.csv' ).to_pandas()
elif mode == 'test':
data = dt.fread(root_dir + 'example_test.csv' ).to_pandas()
elif mode == 'sub':
data = dt.fread(root_dir + 'example_sample_submission.csv' ).to_pandas()
return data
def preprocess_data(data: pd.DataFrame, scale: bool = False, nn: bool = False,
action: str = 'weight'):

 data = data.query('weight > 0' ).reset_index(drop=True)
 data = data.query('date > 85' ).reset_index(drop=True)
 if action == 'weight':
data['action'] =(
(data['weight'].values * data['resp'].values)> 0 ).astype('float32')
 if action == 'combined':
data['action'] =(
(data['resp'].values > 0)and(data['resp_1'].values > 0)and(data['resp_2'].values > 0)and(
 data['resp_3'].values > 0)and(data['resp_4'].values > 0)).astype('float32')
 if action == 'multi':
resp_cols = ['resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4']
for i in range(len(resp_cols)) :
data['action_' + str(i)] =(data['weight'] * data[resp_cols[i]] > 0 ).astype('int')

features = [col for col in data.columns if 'feature' in col] + ['weight']
date = data['date']
if action == 'multi':
target = np.array([data['action_' + str(i)]
for i in range(len(resp_cols)) ] ).T
else:
target = data['action']
data = data[features]
if scale:
scaler = StandardScaler()
data = scaler.fit_transform(data)
 if not scale and nn:
data = data.values
return data, target, features, date
def calc_data_mean(array, cache_dir=None, fold=None, train=True, mode='mean'):
if train:
if mode == 'mean':
f_mean = np.nanmean(array, axis=0)
 if cache_dir and fold:
np.save(f'{cache_dir}/f_{fold}_mean.npy', f_mean)
 elif cache_dir:
np.save(f'{cache_dir}/f_mean.npy', f_mean)
 array = np.nan_to_num(array)+ np.isnan(array)* f_mean
if mode == 'median':
f_med = np.nanmedian(array, axis=0)
 if cache_dir and fold:
np.save(f'{cache_dir}/f_{fold}_median.npy', f_med)
 elif cache_dir:
np.save(f'{cache_dir}/f_median.npy', f_med)
 array = np.nan_to_num(array)+ np.isnan(array)* f_med
if mode == 'zero':
array = np.nan_to_num(array)+ np.isnan(array)* 0
if not train:
if mode == 'mean':
f_mean = np.load(f'{cache_dir}/f_mean.npy')
 array = np.nan_to_num(array)+ np.isnan(array)* f_mean
if mode == 'median':
f_med = np.load(f'{cache_dir}/f_med.npy')
 array = np.nan_to_num(array)+ np.isnan(array)* f_med
if mode == 'zero':
array = np.nan_to_num(array)+ np.isnan(array)* 0
return array
def weighted_mean(scores, sizes):
largest = np.max(sizes)
 weights = [size / largest for size in sizes]
return np.average(scores, weights=weights)
 def create_dataloaders(dataset: Dataset, indexes: dict, batch_size):
train_idx = indexes.get('train', None)
 val_idx = indexes.get('val', None)
 test_idx = indexes.get('test', None)
 dataloaders = {}
if train_idx:
train_set = Subset(
 dataset, train_idx)
 train_sampler = BatchSampler(SequentialSampler(
 train_set), batch_size=batch_size, drop_last=False)
 dataloaders['train'] = DataLoader(
 dataset, sampler=train_sampler, num_workers=10, pin_memory=True)
 if val_idx:
val_set = Subset(dataset, val_idx)
 val_sampler = BatchSampler(SequentialSampler(
 val_set), batch_size=batch_size, drop_last=False)
 dataloaders['val'] = DataLoader(
 dataset, sampler=val_sampler, num_workers=10, pin_memory=True)
 if test_idx:
test_set = Subset(dataset, test_idx)
 test_sampler = BatchSampler(SequentialSampler(
 test_set), batch_size=batch_size, drop_last=False)
 dataloaders['test'] = DataLoader(
 dataset, sampler=test_sampler, num_workers=10, pin_memory=True)
 return dataloaders
def seed_everything(seed):
random.seed(seed)
 torch.manual_seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def load_model(path, input_size, output_size, p, pl_lightning):
if os.path.isdir(path):
models = []
for file in os.listdir(path):
if pl_lightning:
model = Regressor.load_from_checkpoint(checkpoint_path=file, input_size=input_size,
output_size=output_size, params=p)
 else:
model = Regressor(input_size, output_size, params=p)
 model.load_state_dict(torch.load(f'{path}/{file}'))
models.append(model)
 return models
elif os.path.isfile(path):
if pl_lightning:
return Regressor.load_from_checkpoint(checkpoint_path=path, input_size=input_size,
output_size=output_size, params=p)
 else:
model = Regressor(input_size, output_size, params=p)
 model.load_state_dict(torch.load(path))
return model 
<choose_model_class>
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
class Regressor(pl.LightningModule):
def __init__(self, input_size, output_size, params, model_path='models/'):
super(Regressor, self ).__init__()
dim_1 = params['dim_1']
dim_2 = params['dim_2']
dim_3 = params['dim_3']
dim_4 = params['dim_4']
self.dropout_prob = params['dropout']
self.lr = params['lr']
self.activation = params['activation']
self.input_size = input_size
self.output_size = output_size
self.loss = nn.BCEWithLogitsLoss()
self.weight_decay = params['weight_decay']
self.amsgrad = params['amsgrad']
self.label_smoothing = params['label_smoothing']
self.model_path = model_path
self.encoder = nn.Sequential(
 nn.BatchNorm1d(input_size),
nn.Linear(input_size, dim_1, bias=False),
nn.BatchNorm1d(dim_1),
self.activation() ,
nn.Dropout(p=self.dropout_prob),
nn.Linear(dim_1, dim_2, bias=False),
nn.BatchNorm1d(dim_2),
self.activation() ,
nn.Dropout(p=self.dropout_prob),
nn.Linear(dim_2, dim_3, bias=False),
nn.BatchNorm1d(dim_3),
self.activation() ,
nn.Dropout(p=self.dropout_prob),
nn.Linear(dim_3, dim_4, bias=False),
nn.BatchNorm1d(dim_4),
self.activation() ,
nn.Dropout(p=self.dropout_prob),
nn.Linear(dim_4, self.output_size, bias=False)
)
 def forward(self, x):
out = self.encoder(x)
 return out
def training_step(self, batch, batch_idx):
x, y = batch['data'], batch['target']
x = x.view(x.size(1), -1)
 y = y.view(y.size(1), -1)
 logits = self(x)
 loss = self.loss(input=logits, target=y)
 logits = torch.sigmoid(logits)
 auc_metric = roc_auc_score(y_true=y.cpu().numpy() ,
y_score=logits.cpu().detach().numpy())
 self.log('train_auc', auc_metric, on_step=False,
on_epoch=True, prog_bar=True)
 self.log('train_loss', loss, prog_bar=True)
 return {'loss': loss}
def validation_step(self, batch, batch_idx):
x, y = batch['data'], batch['target']
x = x.view(x.size(1), -1)
 y = y.view(y.size(1), -1)
 logits = self(x)
 loss = self.loss(input=logits,
target=y)
 logits = torch.sigmoid(logits)
 auc = roc_auc_score(y_true=y.cpu().numpy() ,
y_score=logits.cpu().detach().numpy())
 return {'loss': loss, 'y': y, 'logits': logits, 'auc': auc}
def validation_epoch_end(self, val_step_outputs):
epoch_loss = torch.tensor([x['loss'] for x in val_step_outputs] ).mean()
epoch_auc = torch.tensor([x['auc'] for x in val_step_outputs] ).mean()
self.log('val_loss', epoch_loss, prog_bar=True)
 self.log('val_auc', epoch_auc, prog_bar=True)
 def test_step(self, batch, batch_idx):
return self.validation_step(batch, batch_idx)
 def test_epoch_end(self, outputs):
epoch_loss = torch.tensor([x['loss'] for x in outputs] ).mean()
epoch_auc = torch.tensor([x['auc'] for x in outputs] ).mean()
self.log('test_loss', epoch_loss)
 self.log('test_auc', epoch_auc)
 def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters() , lr=self.lr,
amsgrad=self.amsgrad)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer, patience=5, factor=0.1, min_lr=1e-7, eps=1e-08 
)
 return {'optimizer': optimizer, 'lr_scheduler': scheduler, 'monitor': 'val_auc'}
def init_weights(m, func):
if type(m)== nn.Linear:
nn.init.xavier_normal_(m.weight, nn.init.calculate_gain(func))
def cross_val(p)-> object:
data_ = load_data(root_dir='./data/', mode='train')
 data_, target_, features, date = preprocess_data(
 data_, nn=True, action='multi')
 input_size = data_.shape[-1]
output_size = target_.shape[-1]
gts = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=5)
 models = []
tb_logger = pl_loggers.TensorBoardLogger('logs/multiclass_')
 for i,(train_idx, val_idx)in enumerate(gts.split(data_, groups=date)) :
idx = np.concatenate([train_idx, val_idx])
 data = copy.deepcopy(data_[idx])
 target = copy.deepcopy(target_[idx])
 checkpoint_callback = pl.callbacks.ModelCheckpoint(
 os.path.join('models/', 'multi_class_fold_{}'.format(i)) , monitor='val_auc', save_top_k=1, period=10,
mode='max' 
)
 model = Regressor(input_size, output_size, p)
 if p['activation'] == nn.ReLU:
model.apply(lambda m: init_weights(m, 'relu'))
elif p['activation'] == nn.LeakyReLU:
model.apply(lambda m: init_weights(m, 'leaky_relu'))
train_idx = [i for i in range(0, max(train_idx)+ 1)]
val_idx = [i for i in range(len(train_idx), len(idx)) ]
data[train_idx] = calc_data_mean(
 data[train_idx], './cache', train=True, mode='mean')
 data[val_idx] = calc_data_mean(
 data[val_idx], './cache', train=False, mode='mean')
 dataset = FinData(data=data, target=target, date=date, multi=True)
 dataloaders = create_dataloaders(
 dataset, indexes={'train': train_idx, 'val': val_idx}, batch_size=p['batch_size'])
 es = EarlyStopping(monitor='val_auc', patience=10,
min_delta=0.0005, mode='max')
 trainer = pl.Trainer(logger=tb_logger,
max_epochs=1,
gpus=1,
callbacks=[checkpoint_callback, es],
precision=16)
 trainer.fit(
 model, train_dataloader=dataloaders['train'], val_dataloaders=dataloaders['val'])
 torch.save(model.state_dict() , f'models/fold_{i}_state_dict.pth')
 models.append(model)
 return models, features<load_from_csv>
mnist_train.isna().any().any()
def final_train(p, load=False):
data_ = load_data(root_dir='./data/', mode='train',overide='/kaggle/input/jane-street-market-prediction/train.csv')
 data, target, features, date = preprocess_data(data_, nn=True)
 dataset = FinData(data=data, target=target, date=date)
 input_size = data.shape[-1]
output_size = 1
train_idx, val_idx = date[date <= 498].index.values.tolist() , date[date > 498].index.values.tolist()
data[train_idx] = calc_data_mean(data[train_idx], './cache', train=True)
 data[val_idx] = calc_data_mean(data[val_idx], './cache', train=False)
 checkpoint_callback = pl.callbacks.ModelCheckpoint(filepath='./cache/full_train',
monitor="val_auc", mode='max', save_top_k=1, period=10)
 model = Regressor(input_size=input_size,
output_size=output_size, params=p)
 if p['activation'] == nn.ReLU:
model.apply(lambda m: init_weights(m, 'relu'))
elif p['activation'] == nn.LeakyReLU:
model.apply(lambda m: init_weights(m, 'leaky_relu'))
dataset = FinData(data, target, date)
 dataloaders = create_dataloaders(dataset, indexes={'train': train_idx, 'val': val_idx}, batch_size=p['batch_size'])
 es = EarlyStopping(monitor='val_auc', patience=10,
min_delta=0.0005, mode='max')
 trainer = pl.Trainer(max_epochs=25,
gpus=1,
callbacks=[checkpoint_callback, es],
precision=16)
 trainer.fit(model, train_dataloader=dataloaders['train'], val_dataloaders=dataloaders['val'])
 torch.save(model.state_dict() , './cache/final_train.pth')
 return model, features<categorify>
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0
def fillna_npwhere(array, values):
if np.isnan(array.sum()):
array = np.nan_to_num(array)+ np.isnan(array)* values
return array
def test_model(models, features, cache_dir='cache'):
env = janestreet.make_env()
iter_test = env.iter_test()
if type(models)== list:
models = [model.eval() for model in models]
else:
models.eval()
f_mean = np.load(f'{cache_dir}/f_mean.npy')
 for(test_df, sample_prediction_df)in tqdm(iter_test):
if test_df['weight'].item() > 0:
vals = torch.FloatTensor(
 fillna_npwhere(test_df[features].values, f_mean))
if type(models)== list:
preds = [torch.sigmoid(model.forward(vals.view(1, -1)) ).detach().numpy()
for model in models]
pred = np.mean(np.mean(preds, axis=1))
else:
pred = torch.sigmoid(models.forward(vals.view(1, -1)) ).item()
sample_prediction_df.action = np.where(
 pred > 0.502, 1, 0 ).astype(int ).item()
else:
sample_prediction_df.action = 0
env.predict(sample_prediction_df )<init_hyperparams>
standardized_scalar = StandardScaler()
standardized_data = standardized_scalar.fit_transform(mnist_train_data)
 standardized_data.shape
def main(train=True):

 p = {'dim_1': 167,
'dim_2': 454,
'dim_3': 371,
'dim_4': 369,
'dim_5': 155,
'activation': nn.LeakyReLU,
'dropout': 0.21062362698532755,
'lr': 0.0022252024054478523,
'label_smoothing': 0.05564974140461841,
'weight_decay': 0.04106097088288333,
'amsgrad': True,
'batch_size': 10072}
if train:
models, features = final_train(p, load=False)
 else:
data_ = load_data(root_dir='./data/', mode='train',overide='/kaggle/input/jane-street-market-prediction/train.csv')
 data_, target_, features, date = preprocess_data(data_, nn=True)
 model_path = '/kaggle/input/model-states'
f_mean = calc_data_mean(data_, 'cache')
 models = load_model(model_path, data_.shape[-1], 5, p, False)
 test_model(models, features)
 return models 
<define_variables>
cov_matrix = np.matmul(standardized_data.T, standardized_data)
 cov_matrix.shape
pca_components = 60<choose_model_class>
lambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))
vectors.shape
e_size = 64
fc_input = pca_components
h_dims = [512,512,256,128]
dropout_rate = 0.5
epochs = 200
minibatch_size = 100000
class MarketPredictor(nn.Module):
def __init__(self):
super(MarketPredictor, self ).__init__()
self.e = nn.Embedding(2,e_size)
 self.deep = nn.Sequential(
 nn.Linear(fc_input,h_dims[0]),
nn.BatchNorm1d(h_dims[0]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[0],h_dims[1]),
nn.BatchNorm1d(h_dims[1]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[1],h_dims[2]),
nn.BatchNorm1d(h_dims[2]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[2],h_dims[3]),
nn.BatchNorm1d(h_dims[3]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[3],e_size),
nn.BatchNorm1d(e_size),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate)
)
 self.reduce = nn.utils.weight_norm(nn.Linear(e_size,5))
def forward(self,xi,xf):
e_out = self.e(xi)
 f_out = self.deep(xf)
 ef_out = self.reduce(e_out+f_out)
 return ef_out 
<load_pretrained>
new_coordinates = np.matmul(vectors, standardized_data.T)
 print(new_coordinates.shape)
 new_coordinates = np.vstack(( new_coordinates, mnist_train_label)).T
epochs = 200
path = '/kaggle/input/pytorch-nn-model-more-feature-engineering/marketpredictor_state_dict_'+str(epochs)+'epochs.pt'
model = MarketPredictor()
model.load_state_dict(torch.load(path,map_location=dev))
model.to(dev)
 model.eval()<load_pretrained>
df_new = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_new.head()
with open('/kaggle/input/pytorch-nn-model-more-feature-engineering/feature_processing.pkl', 'rb')as f:
sc, pca, maxindex, fill_val, remove_names= pickle.load(f )<define_variables>
pca = decomposition.PCA()
pca.n_components = 2
pca_data = pca.fit_transform(standardized_data)
 pca_data.shape
feature_names = ['feature_'+str(i)for i in range(1,130)]
exclude = np.where([maxindex[i,1] > 100 and maxindex [i,2] > 1 for i in range(129)])[0]
keep = np.where([(feature_names[i] != remove_names ).all() for i in range(129)])[0]<split>
pca_data = np.vstack(( pca_data.T, mnist_train_label)).T
env = janestreet.make_env()
iter_test = env.iter_test()<data_type_conversions>
df_PCA = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_PCA.head()
for(test_df, sample_prediction_df)in iter_test:
if test_df['weight'].item() == 0:
sample_prediction_df.action = 0
else:
test_df_features = test_df[feature_names].to_numpy()
for i in exclude:
if test_df_features[0,i] == maxindex[i,0]:
test_df_features[0,i] = fill_val[i]
test_df_int_features = test_df['feature_0'].to_numpy()
nans = np.isnan(test_df_features)
 for i in range(129):
if nans[0,i]:
test_df_features[0,i] = fill_val[i]
test_df_features = test_df_features[:,keep]
test_df_features_scaled = sc.transform(test_df_features)
 test_df_features_pca=pca.transform(test_df_features_scaled)
 itensor = torch.tensor(( test_df_int_features+1)//2,dtype=torch.long,device=dev)
 ftensor = torch.tensor(test_df_features_pca,dtype=torch.float,device=dev)
 s = model(itensor,ftensor)[0,0].item()
sample_prediction_df.action = int(np.round(1/(1+np.exp(-s))))
env.predict(sample_prediction_df)
<choose_model_class>
mnist_train_data = np.array(mnist_train_data)
 mnist_train_label = np.array(mnist_train_label )
e_size = 64
fc_input = 130
h_dims = [512,512,256,128]
dropout_rate = 0.5
epochs = 2000
minibatch_size = 100000
class MarketPredictor(nn.Module):
def __init__(self):
super(MarketPredictor, self ).__init__()
self.deep = nn.Sequential(
 nn.Linear(fc_input,h_dims[0]),
nn.BatchNorm1d(h_dims[0]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[0],h_dims[1]),
nn.BatchNorm1d(h_dims[1]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[1],h_dims[2]),
nn.BatchNorm1d(h_dims[2]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[2],h_dims[3]),
nn.BatchNorm1d(h_dims[3]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[3],e_size),
nn.BatchNorm1d(e_size),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate)
)
 self.reduce = nn.utils.weight_norm(nn.Linear(e_size,5))
def forward(self,xf):
f_out = self.deep(xf)
 ef_out = self.reduce(f_out)
 return ef_out 
<load_pretrained>
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D
from tensorflow.keras.optimizers import Adadelta
from keras.utils.np_utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import LearningRateScheduler
path = '/kaggle/input/pytorch-nn-model-w-o-feature-reduction/marketpredictor_state_dict_'+str(epochs)+'epochs.pt'
model = MarketPredictor()
model.load_state_dict(torch.load(path,map_location=dev))
model.to(dev)
 model.eval()<load_pretrained>
nclasses = mnist_train_label.max() - mnist_train_label.min() + 1
mnist_train_label = to_categorical(mnist_train_label, num_classes = nclasses)
 print("Shape of ytrain after encoding: ", mnist_train_label.shape )
with open('/kaggle/input/pytorch-nn-model-w-o-feature-reduction/feature_processing.pkl', 'rb')as f:
sc, maxindex, fill_val = pickle.load(f )<define_variables>
def build_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def compile_model(model, optimizer='adam', loss='categorical_crossentropy'):
model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
 def train_model(model, train, test, epochs, split):
history = model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)
 return history
feature_names = ['feature_'+str(i)for i in range(130)]
exclude = np.where([maxindex[i,1] > 100 and maxindex [i,2] > 1 for i in range(129)])[0]<split>
cnn_model = build_model(( 28, 28, 1))
compile_model(cnn_model, 'adam', 'categorical_crossentropy')
 model_history = train_model(cnn_model, mnist_train_data, mnist_train_label, 80, 0.2 )
env = janestreet.make_env()
iter_test = env.iter_test()<data_type_conversions>
predictions = cnn_model.predict(mnist_test_arr )
for(test_df, sample_prediction_df)in iter_test:
if test_df['weight'].item() == 0:
sample_prediction_df.action = 0
else:
test_df_features = test_df[feature_names].to_numpy()
for i in exclude:
if test_df_features[0,i+1] == maxindex[i,0]:
test_df_features[0,i+1] = fill_val[i]
nans = np.isnan(test_df_features)
 for i in range(130):
if nans[0,i]:
test_df_features[0,i] = fill_val[i]
test_df_features = sc.transform(test_df_features)
 ftensor = torch.tensor(test_df_features,dtype=torch.float,device=dev)
 s = model(ftensor)[0,0].item()
sample_prediction_df.action = int(np.round(1/(1+np.exp(-s))))
env.predict(sample_prediction_df )<import_modules>
predictions_test = []
for i in predictions:
predictions_test.append(np.argmax(i))
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
from keras.callbacks import ModelCheckpoint
import tensorflow as tf
import numpy as np
import pandas as pd
from tqdm import tqdm
from random import choices
import dask.dataframe as dd
from glob import glob
import os<choose_model_class>
submission = pd.DataFrame({
"ImageId": mnist_test.index+1,
"Label": predictions_test
})
 submission.to_csv('my_submission.csv', index=False )