Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
%matplotlib inline
np.random.seed(17)
 sns.set(style='white', context='notebook', palette='deep')
 for dirname, _, filenames in os.walk('/kaggle/input/digit-recognizer'):
for filename in filenames:
print(os.path.join(dirname, filename))
HIDDEN_LAYER_1 = [256, 256]
HIDDEN_LAYER_2 = [160, 160, 160]
HIDDEN_LAYER_3 = [128, 128, 128, 128]
TARGET_NUM = 5
input = tf.keras.layers.Input(shape=(X_train.shape[1],))
x1 = tf.keras.layers.BatchNormalization()(input)
 x1 = tf.keras.layers.Dropout(0.25 )(x1)
 for units in HIDDEN_LAYER_1:
x1 = tf.keras.layers.Dense(units )(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation(tf.keras.activations.swish )(x1)
 x1 = tf.keras.layers.Dropout(0.25 )(x1)
 x2 = tf.keras.layers.BatchNormalization()(input)
 x2 = tf.keras.layers.Dropout(0.25 )(x2)
 for units in HIDDEN_LAYER_2:
x2 = tf.keras.layers.Dense(units )(x2)
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x2)
 x2 = tf.keras.layers.Dropout(0.25 )(x2)
 x3 = tf.keras.layers.BatchNormalization()(input)
 x3 = tf.keras.layers.Dropout(0.25 )(x3)
 for units in HIDDEN_LAYER_3:
x3 = tf.keras.layers.Dense(units )(x3)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x3 = tf.keras.layers.Activation(tf.keras.activations.swish )(x3)
 x3 = tf.keras.layers.Dropout(0.25 )(x3)
 x = tf.keras.layers.concatenate([x1, x2, x3])
 x = tf.keras.layers.Dense(TARGET_NUM )(x)
 output = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=input, outputs=output)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=1e-3),
metrics = tf.keras.metrics.AUC(name="AUC"),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-2),
)<train_model>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
history = model.fit(
 x = X_train,
y = y_train,
epochs=25,
batch_size=4096,
validation_data=(X_valid, y_valid), 
)
 models = []
models.append(model )<find_best_model_class>
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 sns.countplot(Y_train)
 plt.show()
THRESHOLD = 0.502
janestreet.make_env.__called__ = False
env = janestreet.make_env()
print('predicting...')
 for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
X_test = test_df.loc[:, features].values
if np.isnan(X_test.sum()):
X_test = np.nan_to_num(X_test)+ np.isnan(X_test)* f_mean.values
pred = model(X_test, training = False ).numpy()
pred = np.mean(pred)
 pred_df.action = np.where(pred >= THRESHOLD, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_from_csv>
X_train = X_train / 255.0
test = test / 255.0
SEED = 1111
np.random.seed(SEED)
 train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 cols_to_remove = ['feature_26', 'feature_36', 'feature_24', 'feature_34', 'feature_12', 'feature_22', 'feature_32', 'feature_8', 'feature_18', 'feature_28', 'feature_108', 'feature_114', 'feature_101', 'feature_113', 'feature_107', 'feature_119', 'feature_4', 'feature_6', 'feature_15', 'feature_23', 'feature_25', 'feature_35', 'feature_33', 'feature_38', 'feature_40', 'feature_61', 'feature_66', 'feature_63', 'feature_68', 'feature_76', 'feature_88', 'feature_100', 'feature_112', 'feature_125', 'feature_129', 'feature_82', 'feature_106', 'feature_48', 'feature_57', 'feature_122', 'feature_126', 'feature_128', 'feature_124']
for col in cols_to_remove:
del train[col]
print(train)
<prepare_x_and_y>
X_train = X_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1 )
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=2)
 clf.save(f'model.h5')
 th = 0.502
models = [clf]
f = np.median
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<train_model>
Y_train = to_categorical(Y_train, num_classes = 10 )
Image(".. /input/tf-model-garden-official-models/TF.png" )<import_modules>
random_seed = 2
import tensorflow as tf
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow_addons as tfa
from sklearn.model_selection import StratifiedKFold
import numpy as np
import pandas as pd
from tqdm import tqdm
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed )
NFOLDS = 5
train_all = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train_all = train_all[train_all.date > 85].reset_index(drop = True)
 train_all = train_all[train_all['weight'] != 0]
train_all.fillna(train_all.mean() ,inplace=True )<prepare_x_and_y>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.summary()
train_all['date_bin'] =(pd.qcut(train_all['date'], q=4, labels=False)+1)*train_all['feature_0']
features = [c for c in train_all.columns if "feature" in c]
f_mean = np.mean(train_all[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train_all.loc[:, train_all.columns.str.contains('feature|date_bin')]
y_train = pd.DataFrame(np.stack([(train_all[c] > 0 ).astype('int')for c in resp_cols] ).T, columns = resp_cols )<split>
plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)
 Image("model.png" )
skf = StratifiedKFold(n_splits=NFOLDS, shuffle = True, random_state = 42)
 result = next(skf.split(X_train, X_train.date_bin), None)
 train = train_all.iloc[result[0]].reset_index(drop=True)
 valid = train_all.iloc[result[1]].reset_index(drop=True )<drop_column>
model.compile(optimizer = 'nadam' , loss = "categorical_crossentropy", metrics=["accuracy"] )
del train, valid, train_all, result<choose_model_class>
callbacks_list = [
ReduceLROnPlateau(
 monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=1e-05),
ModelCheckpoint(
 filepath='MNIST_CNN_model.h5',
monitor='val_accuracy',
save_best_only=True 
)]
MNAME = 'model'
def get_callbacks(idx):
mc = ModelCheckpoint(MNAME+"-{}.h5".format(idx), save_best_only=True)
 rp = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.00001)
 es = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=False)
 return [mc, rp, es]
def create_dnn(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 return model<prepare_x_and_y>
epochs = 50
batch_size = 512
skf = StratifiedKFold(n_splits=NFOLDS, shuffle = True, random_state = 42)
 history = []
for i in range(NFOLDS):
print('fold {}'.format(i))
result = next(skf.split(X_train, X_train.date_bin), None)
 X_tr = X_train.iloc[result[0]].reset_index(drop=True)
 X_tr.drop(labels='date_bin', axis = 1, inplace=True)
 y_tr = y_train.iloc[result[0]].reset_index(drop=True)
 X_val = X_train.iloc[result[1]].reset_index(drop=True)
 X_val.drop(labels='date_bin', axis=1, inplace=True)
 y_val = y_train.iloc[result[1]].reset_index(drop=True)
 np.random.seed(42*i)
 tf.keras.backend.clear_session()
tf.random.set_seed(42*i)
 batch_size = 8192
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_dnn(len(features), y_train.shape[1], hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 callbacks = get_callbacks(i)
 epochs = 200
history.append(clf.fit(X_tr, y_tr, epochs=epochs, batch_size=batch_size, validation_data=(X_val,y_val), callbacks=callbacks, verbose=0))
del clf, X_tr, y_tr, X_val, y_val, result<set_options>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
env = janestreet.make_env()<correct_missing_values>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=callbacks_list )
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array<load_pretrained>
model = load_model('MNIST_CNN_model.h5' )
th = 0.501
clf0 = tf.keras.models.load_model("model-0.h5")
 clf2 = tf.keras.models.load_model("model-2.h5")
 clf4 = tf.keras.models.load_model("model-4.h5")
 models = [clf0, clf2, clf4]
test_df_columns = ['weight'] + [f'feature_{i}' for i in range(130)] + ['date']
index_features = [n for n,col in enumerate(test_df_columns)if col in features]
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].values[0] > 0:
x_tt = test_df.values[0][index_features].reshape(1,-1)
 x_tt[:, 1:] = fillna_npwhere_njit(x_tt[:, 1:][0], f_mean)
 pred = np.median(np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0))
pred_df.action = int(pred >= th)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<load_from_csv><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )