Kaggle competition: Digit Recognizer
Submission['Id']=test_data['Id']
Submission.set_index('Id',inplace=True )<save_to_csv>
root = Path('.. /input')
 train_path = Path('train')
 rseed = 7
val_size = 0.05
Submission.to_csv('Submission.csv' )<load_from_csv>
def save_imgs(path:Path, data, labels):
path.mkdir(parents=True,exist_ok=True)
 for label in np.unique(labels): 
(path/str(label)).mkdir(parents=True,exist_ok=True)
 for i in range(len(data)) :
if(len(labels)!=0):
imageio.imsave(str(path/str(labels[i])/(str(i)+'.jpg')) , data[i])
 else:
imageio.imsave(str(path/(str(i)+'.jpg')) , data[i] )
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
batch_size = 5000
hidden_units = [150.5, 150.5, 150.5]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.fit(X_train, y_train, epochs=200, batch_size=5000)
 models = []
models.append(clf)
 th = 0.502
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<define_variables>
train_csv = pd.read_csv(root/'train.csv' )
pca_components = 60<choose_model_class>
test_csv = pd.read_csv(root/'test.csv' )
e_size = 64
fc_input = pca_components
h_dims = [512,512,256,128]
dropout_rate = 0.5
epochs = 200
minibatch_size = 100000
class MarketPredictor(nn.Module):
def __init__(self):
super(MarketPredictor, self ).__init__()
self.e = nn.Embedding(2,e_size)
 self.deep = nn.Sequential(
 nn.Linear(fc_input,h_dims[0]),
nn.BatchNorm1d(h_dims[0]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[0],h_dims[1]),
nn.BatchNorm1d(h_dims[1]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[1],h_dims[2]),
nn.BatchNorm1d(h_dims[2]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[2],h_dims[3]),
nn.BatchNorm1d(h_dims[3]),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate),
nn.Linear(h_dims[3],e_size),
nn.BatchNorm1d(e_size),
nn.LeakyReLU() ,
nn.Dropout(dropout_rate)
)
 self.reduce = nn.utils.weight_norm(nn.Linear(e_size,1))
self.sig = nn.Sigmoid()
def forward(self,xi,xf):
e_out = self.e(xi)
 f_out = self.deep(xf)
 ef_out = self.reduce(e_out+f_out)
 sig_out = self.sig(ef_out)
 return sig_out 
<load_pretrained>
data_X, data_y = train_csv.loc[:,'pixel0':'pixel783'], train_csv['label']
epochs = 200
path = '/kaggle/input/pytorch-nn-model/marketpredictor_state_dict_'+str(epochs)+'epochs.pt'
model = MarketPredictor()
model.load_state_dict(torch.load(path,map_location=dev))
model.to(dev)
 model.eval()<load_pretrained>
train_X, val_X, train_y, val_y = train_test_split(data_X, data_y, test_size=val_size,random_state=rseed,stratify=data_y )
with open('/kaggle/input/pytorch-nn-model/feature_processing.pkl', 'rb')as f:
sc, pca, maxindex, fill_val = pickle.load(f )<define_variables>
def to_img_shape(data_X, data_y=[]):
data_X = np.array(data_X ).reshape(-1,28,28)
 data_X = np.stack(( data_X,)*3, axis=-1)
 data_y = np.array(data_y)
 return data_X,data_y
feature_names = ['feature_'+str(i)for i in range(1,130)]
exclude = np.where([maxindex[i,1] > 100 and maxindex [i,2] > 1 for i in range(129)])[0]<split>
train_X,train_y = to_img_shape(data_X, data_y )
env = janestreet.make_env()
iter_test = env.iter_test()<data_type_conversions>
val_X,val_y = to_img_shape(val_X,val_y )
for(test_df, sample_prediction_df)in iter_test:
if test_df['weight'].item() == 0:
sample_prediction_df.action = 0
else:
test_df_features = test_df[feature_names].to_numpy()
for i in exclude:
if test_df_features[0,i] == maxindex[i,0]:
test_df_features[0,i] = fill_val[i]
test_df_int_features = test_df['feature_0'].to_numpy()
nans = np.isnan(test_df_features)
 for i in range(129):
if nans[0,i]:
test_df_features[0,i] = fill_val[i]
test_df_features_scaled = sc.transform(test_df_features)
 test_df_features_pca=pca.transform(test_df_features_scaled)
 itensor = torch.tensor(( test_df_int_features+1)//2,dtype=torch.long,device=dev)
 ftensor = torch.tensor(test_df_features_pca,dtype=torch.float,device=dev)
 s = model(itensor,ftensor ).item()
sample_prediction_df.action = int(np.round(s))
env.predict(sample_prediction_df)
<import_modules>
save_imgs(Path('/data/train'),train_X,train_y )
import numpy as np
import pandas as pd
from tensorflow.keras.callbacks import TensorBoard
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder<import_modules>
save_imgs(Path('/data/valid'),val_X,val_y )
from tensorflow import keras
from tensorflow.keras.layers import MaxPooling1D, Dense, LeakyReLU, Conv1D
from tensorflow.keras.layers import Flatten, Activation, BatchNormalization, Dropout
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras import layers
from kerastuner.tuners import RandomSearch
from kerastuner.engine.hyperparameters import HyperParameters
import time
import pickle<import_modules>
data = ImageDataBunch.from_folder('/data/',bs=256,size=28,ds_tfms=get_transforms(do_flip=False),num_workers=0 ).normalize(imagenet_stats )
import tensorflow as tf<load_from_csv>
data.show_batch(3,figsize=(6,6))
%%time
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True )<prepare_x_and_y>
learn = cnn_learner(data,models.resnet18,metrics=accuracy,path='.')
 learn.lr_find()
learn.recorder.plot()
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train.loc[:, train.columns.str.contains('feature')]
Y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
leaky_relu_alpha =0.05
LeakyReLU(alpha=leaky_relu_alpha)
<choose_model_class>
learn.fit_one_cycle(1,1e-02 )
def build_model() :
model = keras.models.Sequential()
model.add(Conv1D(180, 2, input_shape=x_train.shape[1:]))
model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=leaky_relu_alpha))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.15))
model.add(Flatten())
 model.add(Dense(180))
model.add(LeakyReLU(alpha=leaky_relu_alpha))
model.add(Dropout(0.15))
model.add(Dense(5))
model.add(Activation("sigmoid"))
model.compile(optimizer=keras.optimizers.Adam(lr=1e-3),
loss=BinaryCrossentropy(label_smoothing=0.095),
metrics=[tf.keras.metrics.AUC(name = 'auc'), "accuracy"])
 return model<choose_model_class>
learn.save('s1' )
model = build_model()<train_model>
learn.load('s1');
model.fit(x=x_train,
y=Y,
epochs=10,
batch_size=1024 )<import_modules>
learn.lr_find()
from tqdm import tqdm<feature_engineering>
learn.fit_one_cycle(10,max_lr=slice(1e-6,1e-5))
f = np.median
th = 0.5000
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt.reshape(-1, x_tt.shape[1], 1), training = False ).numpy() ],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<define_variables>
interp = ClassificationInterpretation.from_learner(learn )
START_TIME = time.time() 
<data_type_conversions>
learn1 = learn.load('s1')
 sub_df = pd.DataFrame(columns=['ImageId','Label'] )
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.median() , inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(train['resp'] > 0 ).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_median = np.median(train[features[1:]].values, axis=0)
<prepare_x_and_y>
def get_img(data):
t1 = data.reshape(28,28)/255
t1 = np.stack([t1]*3,axis=0)
 img = Image(FloatTensor(t1))
return img
y_resps = train[resp_cols].values
y_actions = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<choose_model_class>
from fastprogress import progress_bar
def create_model(input_dim, output_dims, add_models=0):
input_layer_0 = Input(input_dim)
 bn_0 = BatchNormalization()(input_layer_0)
 outputs_layer_0 = []
for m in range(2+add_models):
x = Dropout(0.2 )(bn_0)
 for i in range(m+1):
x = Dense(64 )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(0.1 )(x)
 output = Dense(output_dims[3], activation='linear', name=f'level_0_output_{m}' )(x)
 outputs_layer_0.append(output)
 output_layer_0_average = Average(name='output_layer_0_average' )(outputs_layer_0)
 bn_1 = BatchNormalization()(output_layer_0_average)
 input_layer_1 = Concatenate()([bn_0] + [bn_1])
 outputs_layer_1 = []
for m in range(2+add_models):
x = Dropout(0.2 )(input_layer_1)
 for i in range(m+1):
x = Dense(64 )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(0.1 )(x)
 output = Dense(output_dims[2], activation='sigmoid', name=f'level_1_output_{m}' )(x)
 outputs_layer_1.append(output)
 output_layer_1_average = Average(name='output_layer_1_average' )(outputs_layer_1)
 bn_2 = BatchNormalization()(output_layer_1_average)
 input_layer_2 = Concatenate()([bn_1] + [bn_2])
 outputs_layer_2 = []
for m in range(2+add_models):
x = Dropout(0.2 )(input_layer_2)
 for i in range(m+1):
x = Dense(64 )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(0.1 )(x)
 output = Dense(output_dims[1], activation='linear', name=f'level_2_output_{m}' )(x)
 outputs_layer_2.append(output)
 output_layer_2_average = Average(name='output_layer_2_average' )(outputs_layer_2)
 bn_3 = BatchNormalization()(output_layer_2_average)
 input_layer_3 = Concatenate()([bn_2] + [bn_3])
 outputs_layer_3 = []
for m in range(2+add_models):
x = Dropout(0.2 )(input_layer_3)
 for i in range(m+1):
x = Dense(64 )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(0.1 )(x)
 output = Dense(output_dims[1], activation='sigmoid', name=f'level_3_output_{m}' )(x)
 outputs_layer_3.append(output)
 output_layer_3_average = Average(name='output_layer_3_average' )(outputs_layer_3)
 model = Model(inputs=input_layer_0, outputs=[output_layer_3_average, output_layer_2_average, output_layer_1_average, output_layer_0_average])
 loss = {}
loss['output_layer_3_average'] = BinaryCrossentropy(label_smoothing = 0.1)
 loss['output_layer_2_average'] = 'mse'
loss['output_layer_1_average'] = BinaryCrossentropy(label_smoothing = 0.1)
 loss['output_layer_0_average'] = 'mse'
loss_weights={}
loss_weights['output_layer_3_average'] =.25
loss_weights['output_layer_2_average'] =.25
loss_weights['output_layer_1_average'] =.25
loss_weights['output_layer_0_average'] =.25
metrics = {}
metrics['output_layer_3_average'] = tf.keras.metrics.AUC(name = 'auc')
 metrics['output_layer_2_average'] = 'mse'
metrics['output_layer_1_average'] = tf.keras.metrics.AUC(name = 'auc')
 metrics['output_layer_0_average'] = 'mse'
model.compile(optimizer = Adam() , loss = loss, metrics = metrics, loss_weights=loss_weights)
 return model 
<prepare_x_and_y>
sub_df.to_csv('submission.csv',index=False )