Kaggle competition: Digit Recognizer
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model<prepare_x_and_y>
train_df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test_df = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 submission_df = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv" )
data = dd.read_parquet('.. /input/janestreetparquetdata/date*.parquet')
 features = ['feature_{}'.format(i)for i in range(130)]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
train = data.compute()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
f_mean = train[features].mean().values
train = train.dropna()
X_train = train[features].values
y_train =(train[resp_cols]> 0 ).astype(int ).values<init_hyperparams>
X_train = train_df.iloc[:, 1:].values
y_train = train_df.iloc[:, 0].values
X_test = test_df.values
print(f"X_train shape: {X_train.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"X_test shape: {X_test.shape}" )
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 hidden_units = [150, 150, 150]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
epochs = 250
batch_size = 5000
save_every_n_epochs = 10
save_freq =(len(X_train)//batch_size)*save_every_n_epochs
clf = create_mlp(
 len(features), len(resp_cols), hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 checkpoint_path = "./cp-{epoch:04d}.ckpt"
checkpoint = ModelCheckpoint(checkpoint_path,save_weights_only=True,save_freq=save_freq)
 clf.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,callbacks=[checkpoint])
 clf.save_weights('./SimpleMLP.ckpt' )<define_search_space>
X_train_combined = np.r_[X_train, X_train_add]
y_train_combined = np.r_[y_train, y_train_add]
del X_train
del X_train_add
del y_train
del y_train_add
print(f"X_train_combined shape: {X_train_combined.shape}")
 print(f"y_train_combined shape: {y_train_combined.shape}" )
class TrainData(Dataset):
def __init__(self,file_name,root_dir,predict=False):
self.file_name = file_name
self.root_dir = root_dir
self.feature = ['feature_{}'.format(i)for i in range(130)]
self.resp = ['resp_{}'.format(i)for i in range(1,5)]+['resp']
self.prediction = predict
def __len__(self):
return len(glob(os.path.join(self.root_dir,'*.parquet')))
 def __getitem__(self, idx):
data = pd.read_parquet(os.path.join(self.root_dir,self.file_name+'_{}.parquet'.format(idx)))
 X = data[self.feature].values
W = data['weight'].values
R = data['resp'].values
return X,W,R<categorify>
class ImageReshaper(BaseEstimator, TransformerMixin):

 def __init__(self, shape):
self.shape = shape
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return X.reshape(self.shape )
dataset = TrainData('date','.. /input/janestreetparquetdata/',predict = True)
 weight_path = glob('./*.ckpt.index')
 weight_path = [os.path.basename(each ).split('.')[0] for each in weight_path]
weight_path.sort()
for path in weight_path:
clf.load_weights('./{}.ckpt'.format(path))
p = []
for i in range(len(dataset)) :
net_input,weight,resp= dataset[i]
net_input = np.nan_to_num(net_input)+f_mean*(np.isnan(net_input ).astype(int))
net_prediction = clf.predict(net_input)
 pre =(np.median(net_prediction,axis=1)>0.5 ).astype(int)
 p.append(( weight*resp*pre ).sum())
 result = pd.DataFrame(data={'p':p})
 result ['p2'] = result['p']**2
t =(result['p'].sum() /np.sqrt(result['p2'].sum())) *(np.sqrt(250/len(dataset)))
 print(path,min(max(t,0),6)*result['p'].sum() )<load_pretrained>
def build_lenet5_model() :

 model = Sequential()
model.add(Conv2D(6, kernel_size=5, activation='relu',
input_shape=(28,28,1)))
 model.add(MaxPooling2D())
 model.add(Conv2D(16, kernel_size=5, activation='relu'))
model.add(MaxPooling2D())
 model.add(Flatten())
 model.add(Dense(400, activation='relu'))
model.add(Dense(120, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])
 return model
selection = 'cp-0200'
clf.load_weights('./{}.ckpt'.format(selection))<predict_on_test>
def build_custom_lenet5_model() :

 model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])
 return model
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt = np.nan_to_num(x_tt)+f_mean*(np.isnan(x_tt ).astype(int))
pred = np.median(clf(x_tt, training=False))
pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_pretrained>
stratified_fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for fold, indices in enumerate(stratified_fold.split(X_train_combined, y_train_combined)) :
X_train_, y_train_ = X_train_combined[indices[0]], y_train_combined[indices[0]]
X_test_, y_test_ = X_train_combined[indices[1]], y_train_combined[indices[1]]
model_pipeline = Pipeline([ 
('min_max_scaler', MinMaxScaler()), 
('image_reshaper', ImageReshaper(shape=(-1, 28, 28, 1))), 
('model', KerasClassifier(build_lenet5_model, epochs=5, batch_size=32))
])
 model_pipeline.fit(X_train_, y_train_)
 predictions = model_pipeline.predict(X_test_)
 print(f"Classification report for Fold {fold + 1}:")
 print(classification_report(y_test_, predictions, digits=3), end="
")
 print(f"Confusion Matrix for Fold {fold + 1}:")
 print(confusion_matrix(y_test_, predictions), end="
")
 del X_train_
del X_test_
del y_train_
del y_test_
SEED = 1111
inference = False
cv = False
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train_pickle_file = '/kaggle/input/pickling/train.csv.pandas.pickle'
train = pickle.load(open(train_pickle_file, 'rb'))
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 train['bias'] = 1
features = [c for c in train.columns if "feature" in c]<normalization>
lenet5_model = Pipeline([ 
('min_max_scaler', MinMaxScaler()), 
('image_reshaper', ImageReshaper(shape=(-1, 28, 28, 1))), 
('model', KerasClassifier(build_lenet5_model, epochs=5, batch_size=32))
])
 custom_lenet5_model = Pipeline([ 
('min_max_scaler', MinMaxScaler()), 
('image_reshaper', ImageReshaper(shape=(-1, 28, 28, 1))), 
('model', KerasClassifier(build_custom_lenet5_model, epochs=20, batch_size=32))
])
 lenet5_model.fit(X_train_combined, y_train_combined)
 lenet5_model_predictions = lenet5_model.predict_proba(X_test)
 custom_lenet5_model.fit(X_train_combined, y_train_combined)
 custom_lenet5_model_predictions = custom_lenet5_model.predict_proba(X_test )
def build_neutralizer(train, features, proportion, return_neut=False):

 neutralizer = {}
neutralized_features = np.zeros(( train.shape[0], len(features)))
 target = train[['resp', 'bias']].values
for i, f in enumerate(features):
feature = train[f].values.reshape(-1, 1)
 coeffs = np.linalg.lstsq(target, feature)[0]
neutralized_features[:, i] =(feature -(proportion * target.dot(coeffs)) ).squeeze()
neutralizer = np.linalg.lstsq(train[features+['bias']].values, neutralized_features)[0]
if return_neut:
return neutralized_features, neutralizer
else:
return neutralizer
def neutralize_array(array, neutralizer):
neutralized_array = array.dot(neutralizer)
 return neutralized_array
def test_neutralization() :
dummy_train = train.loc[:100000, :]
proportion = 1.0
neutralized_features, neutralizer = build_neutralizer(dummy_train, features, proportion, True)
 dummy_neut_train = neutralize_array(dummy_train[features+['bias']].values, neutralizer)
 print(neutralized_features[0, :10], dummy_neut_train[0, :10])
 test_neutralization()<feature_engineering>
predictions = lenet5_model_predictions + custom_lenet5_model_predictions
predictions = np.argmax(predictions, axis=1 )
<prepare_x_and_y><EOS> 
submission_df["Label"] = predictions
submission_df.to_csv('submissions.csv', index=False)
 FileLink('submissions.csv' )