Kaggle competition: Digit Recognizer
models = []
models.append(clf)
 th = 0.503
f = np.median
f_mean = np.mean(train[features[1:]].values,axis=0)
 env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_from_csv>
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep' )
df = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
df = df.query('date > 85' ).reset_index(drop = True)
 df = df[df.weight > 0]
df.reset_index(inplace=True, drop=True)
 df = df.astype({c: np.float32 for c in df.select_dtypes(include='float64' ).columns})
 df_labels = df[['date', 'weight', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']]
features = [c for c in df.columns if 'feature' in c]
df = df.drop(df_labels.columns, axis=1)
 df.fillna(df.mean() , inplace=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X = df[features]
y = np.stack([(df_labels[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(df[features[1:]].values, axis=0)
 device = 'cuda' if t.cuda.is_available() else 'cpu'<choose_model_class>
train=pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test=pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
class Model(nn.Module):
def __init__(self, input_size):
super(Model, self ).__init__()
hs = 256
self.batch_norm0 = nn.BatchNorm1d(input_size)
 self.layer1 = LinBnDrop(input_size, hs, bn=True, p=0, act=Mish() , lin_first=False)
 self.layer2 = LinBnDrop(hs + input_size, hs, bn=True, p=0.2289, act=Mish() , lin_first=False)
 self.layer3 = LinBnDrop(hs + hs, hs, bn=True, p=0.2289, act=Mish() , lin_first=False)
 self.layer4 = LinBnDrop(hs + hs, hs, bn=True, p=0.2, act=Mish() , lin_first=False)
 self.dense5 = nn.Linear(hs + hs, 5)
 def forward(self, x):
x = self.batch_norm0(x)
 x1 = self.layer1(x)
 x = torch.cat([x, x1], 1)
 x2 = self.layer2(x)
 x = torch.cat([x1, x2], 1)
 x3 = self.layer3(x)
 x = torch.cat([x2, x3], 1)
 x4 = self.layer4(x)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x<load_pretrained>
def load_data() :

 path='/kaggle/input/mnist-numpy/mnist.npz'
f = np.load(path)
 x_train, y_train = f['x_train'], f['y_train']
x_test, y_test = f['x_test'], f['y_test']
f.close()
return(x_train, y_train),(x_test, y_test )
model_nn = Model(len(features))
model_nn = model_nn.to(device)
 learn = Learner(None, model_nn, loss_func=1)
 learn.load('/kaggle/input/roclossjs3/dense_model')
<categorify>
Y_train=train["label"]
X_train=train.drop("label",axis=1)
(x_train1,y_train1),(x_test1,y_test1)=load_data()
train1=np.concatenate([x_train1,x_test1],axis=0)
 y_train1=np.concatenate([y_train1,y_test1],axis=0)
 Y_train1=y_train1
X_train1=train1.reshape(-1,28*28 )
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
def for_loop(method, matrix, values):
for i in range(matrix.shape[0]):
matrix[i] = method(matrix[i], values)
 return matrix<find_best_params>
X_train=X_train/255.0
test=test/255.0
X_train1=X_train1/255.0
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )<compute_test_metric>
X_train = np.concatenate(( X_train.values, X_train1))
Y_train = np.concatenate(( Y_train, Y_train1))
X_train=X_train.reshape(-1,28,28,1)
 test=test.values.reshape(-1,28,28,1)
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5 )<import_modules>
Y_train = to_categorical(Y_train,num_classes=10)
tf.__version__<define_variables>
random_seed=2
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
<load_from_csv>
X_train,X_val,Y_train,Y_val=train_test_split(X_train,Y_train,test_size=0.25,random_state=random_seed)
%%time
train=pd.read_parquet('.. /input/step01-csv-parquet/dtrain.parquet')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train['action'] =(( train['resp'].values)> 0 ).astype(int)
 train.fillna(train.mean() ,inplace=True)
 raw_features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[raw_features[1:]].values,axis=0)
 np.save("f_mean_online.npy",f_mean)
 train.shape<feature_engineering>
model=Sequential()
model.add(Conv2D(filters=64,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=64,kernel_size=(5,5),padding='Same',activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 128, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 128, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(10, activation = "softmax"))
def stats_features(tmp_df):
tmp_df['feature_cross_41_42_43']=tmp_df['feature_41']+tmp_df['feature_42']+tmp_df['feature_43']
tmp_df['feature_cross_1_2']=tmp_df['feature_1']/(tmp_df['feature_2']+1e-5)
 tmp_df.head()
return tmp_df 
<feature_engineering>
plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)
 Image("model.png" )
train=stats_features(train)
 train.head()<define_variables>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
features = [c for c in train.columns if "feature" in c]<prepare_x_and_y>
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )
valid = train.loc[(train.date >= 450)&(train.date < 500)].reset_index(drop=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
X_valid = valid.loc[:, valid.columns.str.contains('feature')]
y_valid = np.stack([(valid[c] > 0 ).astype('int')for c in resp_cols] ).T
del train<choose_model_class>
learning_rate_reduction=ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
def create_resnet(
 num_columns, num_labels, hidden_size, dropout_rate, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x=tf.keras.layers.BatchNormalization()(inp)
 x=tf.keras.layers.Dropout(dropout_rate )(x)
 x1=tf.keras.layers.Dense(hidden_size )(x)
 x1=tf.keras.layers.BatchNormalization()(x1)
 x1=tf.keras.layers.LeakyReLU()(x1)
 x1=tf.keras.layers.Dropout(dropout_rate )(x1)
 x = tf.keras.layers.concatenate([x, x1], axis=1)
 x2=tf.keras.layers.Dense(hidden_size )(x)
 x2=tf.keras.layers.BatchNormalization(axis=1 )(x2)
 x2=tf.keras.layers.LeakyReLU()(x2)
 x2=tf.keras.layers.Dropout(dropout_rate )(x2)
 x = tf.keras.layers.concatenate([x1, x2], axis=1)
 x3=tf.keras.layers.Dense(hidden_size )(x)
 x3=tf.keras.layers.BatchNormalization(axis=1 )(x3)
 x3=tf.keras.layers.LeakyReLU()(x3)
 x3=tf.keras.layers.Dropout(dropout_rate )(x3)
 x = tf.keras.layers.concatenate([x2, x3], axis=1)
 x4=tf.keras.layers.Dense(hidden_size )(x)
 x4=tf.keras.layers.BatchNormalization(axis=1 )(x4)
 x4=tf.keras.layers.LeakyReLU()(x4)
 x4=tf.keras.layers.Dropout(dropout_rate )(x4)
 x = tf.keras.layers.concatenate([x3, x4], axis=1)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(lr=learning_rate,weight_decay=1e-5),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
NUM_FOLDS=5
TRAINING=True
if TRAINING:
for i in range(NUM_FOLDS):
tf.keras.backend.clear_session()
SEED = 1111
tf.random.set_seed(SEED+i)
 np.random.seed(SEED+i)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
verbose=1,
factor=0.2,
patience=8, mode='min')
 earlystop_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath='online_model_{}.weights'.format(i),
save_weights_only=True,
monitor='val_AUC',
mode='max',
verbose=1,
save_best_only=True)
 dropout_rate=0.2
hidden_size=256
label_smoothing = 0.005
learning_rate = 1e-3
clf = create_resnet(
 len(features), 5, hidden_size, dropout_rate, label_smoothing, learning_rate 
)
 clf.summary()
clf.fit(X_train, y_train, epochs=200, batch_size=4096,validation_data=(X_valid,y_valid),
callbacks=[reduce_lr,
earlystop_callback,
model_checkpoint_callback])
 gc.collect() 
<feature_engineering>
epochs = 50
batch_size = 128
<import_modules>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train)
 history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction])
<load_from_csv><EOS> 
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )