Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import keras
import tensorflow
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report,accuracy_score
from keras.models import Sequential
from keras.layers import Convolution2D,MaxPooling2D,Flatten,Dense,Dropout
from keras.utils.np_utils import to_categorical
from keras.datasets import mnist
class UtilityScoreCallback(tf.keras.callbacks.Callback):
def __init__(self, X, date, weight, resp, batch_size = 1024,
early_stopping_patience = 30, plateau_patience = 10, min_lr = 1e-6,
reduction_rate = 0.3, stage = 'train', fold_n = 0, verbose = 1):
super(Callback, self ).__init__()
self.X = X
self.date = date
self.weight = weight
self.resp = resp
self.history = []
self.batch_size = batch_size
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.min_lr = min_lr
self.reduction_rate = reduction_rate
self.fold_n = fold_n
self.stage = stage
self.verbose = verbose
self.best_uscore = - float('inf')
 self.checkpoints_path = f'models_1.0/'
if not os.path.exists(self.checkpoints_path):
os.makedirs(self.checkpoints_path)
 def utility_score(self, d, w, r, a):
Pi = np.bincount(d, w * r * a)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / len(Pi))
u = min(max(t, 0), 6)* np.sum(Pi)
 return u
def is_patience_lost(self, patience):
if len(self.history)> patience:
best_performance = max(self.history[-(patience + 1):-1])
 return best_performance == self.history[-(patience + 1)] and best_performance >= self.history[-1]
def early_stopping_check(self, uscore):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, uscore, epoch):
if uscore > self.best_uscore:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, f'JSModel_{self.fold_n}*')) :
os.remove(checkpoint)
 self.best_uscore = uscore
self.model.save(os.path.join(self.checkpoints_path, f'ResnetModel_{self.fold_n}.hdf5'))
if self.verbose:
print(f"
{'
Saved new checkpoint with Score {self.best_uscore}
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = max(float(K.get_value(self.model.optimizer.lr)) * self.reduction_rate, self.min_lr)
 K.set_value(self.model.optimizer.lr, new_lr)
 if self.verbose:
print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_preds = self.model.predict(self.X, batch_size=self.batch_size)
 action_proba = np.median(y_preds, axis=1)
 action = np.where(action_proba >= 0.5, 1, 0)
 uscore = self.utility_score(self.date, self.weight, self.resp, action)
 self.history.append(uscore)
 if self.stage == 'val':
self.early_stopping_check(uscore)
 self.model_checkpoint(uscore, epoch)
 if self.plateau_patience >= 0:
self.reduce_lr_on_plateau()
def get_uscore_history(self):
return self.history<choose_model_class>
df_train=pd.read_csv('.. /input/digit-recognizer/train.csv')
 df_test=pd.read_csv('.. /input/digit-recognizer/test.csv' )
def create_resnet(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x_1 = tf.keras.layers.Concatenate(axis=-1 )([x, inp])
 for i in range(len(hidden_units)) :
x_1 = tf.keras.layers.Dense(hidden_units[i] )(x_1)
 x_1 = tf.keras.layers.BatchNormalization()(x_1)
 x_1 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_1)
 x_1 = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x_1)
 x_2 = tf.keras.layers.Concatenate(axis=-1 )([x_1, x])
 for i in range(len(hidden_units)) :
x_2 = tf.keras.layers.Dense(hidden_units[i] )(x_2)
 x_2 = tf.keras.layers.BatchNormalization()(x_2)
 x_2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_2)
 x_2 = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x_2)
 x_3 = tf.keras.layers.Concatenate(axis=-1 )([x_1, x_2])
 x_3 = tf.keras.layers.Dense(256 )(x_3)
 x_3 = tf.keras.layers.BatchNormalization()(x_3)
 x_3 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_3)
 x_3 = tf.keras.layers.Dropout(0.2 )(x_3)
 x_3 = tf.keras.layers.Dense(num_labels )(x_3)
 out = tf.keras.layers.Activation("sigmoid" )(x_3)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model<define_search_space>
print(type(df_test))
print(type(df_train))
df_test.isnull().sum()
df_train.isnull().sum()
df_test.isnull().sum().sum()
df_train.isnull().sum().sum()
batch_size = 2048
hidden_units = [150, 150, 150]
dropout_rates = [0.25, 0.25, 0.25, 0.25]
label_smoothing = 1e-3
learning_rate = 1e-3
folds = 5
train_mode = True
opt_th_cross = 0.5<train_model>
classifier = Sequential()
classifier.add(Convolution2D(filters = 128, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 classifier.add(MaxPooling2D(pool_size=(2,2)))
 classifier.add(BatchNormalization())
 classifier.add(Convolution2D(filters = 128, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size=(2,2)))
 classifier.add(Dropout(0.25))
if train_mode:
clf = create_resnet(len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.fit(train.loc[:, features].values,(train.loc[:,resp_cols] > 0 ).astype(int), epochs=150, batch_size=batch_size, shuffle=True)
<categorify>
classifier.add(Convolution2D(filters =256, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size=(2,2)))
 classifier.add(Convolution2D(filters = 256, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 classifier.add(Dropout(0.25))
models = []
clf.call = tf.function(clf.call, experimental_relax_shapes=True)
 models.append(clf )<split>
classifier.add(Flatten())
 classifier.add(Dense(256, activation = "relu"))
classifier.add(Dropout(0.3))
classifier.add(Dense(10, activation = "softmax"))
env = janestreet.make_env()
env_iter = env.iter_test()<feature_engineering>
classifier.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'] )
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].values[0] > 0:
test_df = test_df.loc[:, features].values
if np.isnan(test_df[:, 1:].sum()):
test_df[:, 1:] = np.nan_to_num(test_df[:, 1:])+ np.isnan(test_df[:, 1:])* f_mean
pred = np.mean([model(test_df, training = False ).numpy() for model in models],axis=0)
 pred_df.action = int(0.6*pred[:,3] + 0.2*pred[:,2] + 0.2*pred[:,4]> opt_th_cross)
 else:
pred_df["action"].values[0] = 0
env.predict(pred_df )<set_options>
classifier.fit(target,label_cat,epochs=50 )
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 seed = 42
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
if not os.path.exists("results"):
os.mkdir("results")
 TRAINING = True
read_path = '/kaggle/input/jane-street-market-prediction/train.csv'
save_path = os.path.join("results", str(seed))
device = torch.device("cuda:0")
 if not os.path.exists(save_path):
os.mkdir(save_path)
<define_variables>
results=classifier.predict_classes(df_test)
 print(results)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission.csv",index=False,header=True )
TRAINING = True<data_type_conversions>
( X_train,y_train),(X_test,y_test)=mnist.load_data()
print(X_train.shape)
 print(X_test.shape)
 print(type(X_train))
img=X_train[0]
plt.imshow(img,cmap='gray')
 print(type(img))
train = pd.read_csv(read_path)
 features = [c for c in train.columns if 'feature' in c]
f_mean = train[features].mean()
train = train.loc[train.weight > 0].reset_index(drop = True)
 train[features] = train[features].fillna(f_mean)
 train = train.astype("float32")
 train['action'] =(train['resp'] > 0 ).astype('int')
 train['action1'] =(train['resp_1'] > 0 ).astype('int')
 train['action2'] =(train['resp_2'] > 0 ).astype('int')
 train['action3'] =(train['resp_3'] > 0 ).astype('int')
 train['action4'] =(train['resp_4'] > 0 ).astype('int')
 targets = ['resp']
targets = ['action', 'action1', 'action2', 'action3', 'action4']
def add_features(df, features):
new_features = copy.deepcopy(features)
 df["cross_1_2"] = df["feature_1"] /(df["feature_2"] + 1e-5)
 df["cross_41_42_43"] = df["feature_41"] + df["feature_42"] + df["feature_43"]
new_features.extend(["cross_1_2", "cross_41_42_43"])
 return df, new_features
train, train_features = add_features(train, features)
<filter>
X_train=X_train.reshape(60000,28,28,1)
 X_test=X_test.reshape(10000,28,28,1)
 y_cat_train=to_categorical(y_train,10)
 y_cat_test=to_categorical(y_test,10 )
test = train.loc[(train.date >= 450)&(train.date < 500)].reset_index(drop=True)
<data_type_conversions>
classifier.fit(X_train,y_cat_train,epochs=25 )
class MyDataset:
def __init__(self, df, features, targets):
self.features = df[features].values
self.labels = df[targets].values
self.weights = df['weight'].values
def __len__(self):
return len(self.labels)
 def __getitem__(self, idx):
feat_ = torch.tensor(self.features[idx], dtype=torch.float)
 label_ = torch.tensor(self.labels[idx], dtype=torch.float)
 weight_ = torch.tensor(self.weights[idx], dtype=torch.float)
 return feat_, label_, weight_<choose_model_class>
results=classifier.predict_classes(df_test)
 print(results)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission1.csv",index=False,header=True )
class Model(nn.Module):
def __init__(self, all_feat_cols, target_cols):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.1)
 dropout_rate = 0.1
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(2 * hidden_size, 128)
 self.batch_norm5 = nn.BatchNorm1d(128)
 self.dropout5 = nn.Dropout(dropout_rate)
 self.dense6 = nn.Linear(128, 128)
 self.batch_norm6 = nn.BatchNorm1d(128)
 self.dropout6 = nn.Dropout(dropout_rate)
 self.dense7 = nn.Linear(128, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.RReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.RReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.RReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.RReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 x = self.batch_norm5(x)
 x = self.RReLU(x)
 x = self.dropout5(x)
 x = self.dense6(x)
 x = self.batch_norm6(x)
 x = self.RReLU(x)
 x = self.dropout6(x)
 x = self.dense7(x)
 return x
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets, weights=None):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets, weights)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class EarlyStopping:
def __init__(self, patience=7, mode="max", delta=0.) :
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved.Saving model!')
 torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score<train_model>
classifier.fit(X_test,y_cat_test,epochs=25 )
<split><EOS> 
results=classifier.predict_classes(df_test)
 print(results)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission2.csv",index=False,header=True )