Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<find_best_params>
import numpy as np
import pandas as pd
supported_models = timm.list_models()
print(f"{len(supported_models)} models are supported in timm.")
 print(supported_models )<import_modules>
train_pd = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_pd = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
class EMA(object):

 def __init__(
 self,
model: nn.Module,
decay: float,
strict: bool = True,
use_dynamic_decay: bool = True, 
):
self.decay = decay
self.model = model
self.strict = strict
self.use_dynamic_decay = use_dynamic_decay
self.logger = getLogger(__name__)
 self.n_step = 0
self.shadow = {}
self.original = {}
self._assigned = False
for name, param in model.named_parameters() :
if param.requires_grad:
self.shadow[name] = param.data.clone()
def step(self):
self.n_step += 1
if self.use_dynamic_decay:
_n_step = float(self.n_step)
 decay = min(self.decay,(1.0 + _n_step)/(10.0 + _n_step))
else:
decay = self.decay
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
new_average =(1.0 - decay)* param.data + decay * self.shadow[name]
self.shadow[name] = new_average.clone()
__call__ = step
def assign(self):

 if self._assigned:
if self.strict:
raise ValueError("[ERROR] `assign` is called again before `resume`.")
 else:
self.logger.warning(
 "`assign` is called again before `resume`."
"shadow parameter is already assigned, skip." 
)
 return
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
self.original[name] = param.data.clone()
param.data = self.shadow[name]
self._assigned = True
def resume(self):

 if not self._assigned:
if self.strict:
raise ValueError("[ERROR] `resume` is called before `assign`.")
 else:
self.logger.warning("`resume` is called before `assign`, skip.")
 return
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
param.data = self.original[name]
self._assigned = False 
<set_options>
label_nums = train_pd["label"]
label = to_categorical(label_nums, num_classes = 10)
 train = train_pd.drop(labels = ["label"],axis = 1)/ 255.0
test = test_pd / 255.0 
(rows, cols, channels)=(28,28,1)
 def ReShape(DF, rows, cols, channels):
return DF.values.reshape(DF.shape[0],rows,cols,channels)
 train = ReShape(train, rows, cols, channels)
 test = ReShape(test, rows, cols, channels)
 X_train, X_val, Y_train, Y_val = train_test_split(train,label, test_size = 0.1)
 print("X_train shape:" , X_train.shape, " Y_train shape:", Y_train.shape)
 print("X_val shape:", X_val.shape, " Y_val shape:", Y_val.shape)
 print("test shape:", test.shape )
class LRScheduler(Extension):

 trigger = 1, 'iteration'
priority = PRIORITY_READER
name = None
def __init__(self, optimizer: optim.Optimizer, scheduler_type: str, scheduler_kwargs: Mapping[str, Any])-> None:
super().__init__()
self.scheduler = getattr(optim.lr_scheduler, scheduler_type )(optimizer, **scheduler_kwargs)
 def __call__(self, manager: ExtensionsManager)-> None:
self.scheduler.step()
def state_dict(self)-> None:
return self.scheduler.state_dict()
def load_state_dict(self, to_load)-> None:
self.scheduler.load_state_dict(to_load)
<train_model>
from keras.layers import Activation,Dropout,Dense,Conv2D,AveragePooling2D,MaxPooling2D,Flatten,ZeroPadding2D,BatchNormalization
from tensorflow.keras import optimizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.losses import categorical_crossentropy
from keras.callbacks import EarlyStopping
def create_trainer(model, optimizer, device)-> Engine:
model.to(device)
 def update_fn(engine, batch):
model.train()
optimizer.zero_grad()
loss, metrics = model(*[elem.to(device)for elem in batch])
 loss.backward()
optimizer.step()
return metrics
trainer = Engine(update_fn)
 return trainer 
<import_modules>
class LeNet5(Sequential):
def __init__(self, input_shape=(rows, cols, channels),activation='tanh',pooling='avg',dropout = 0, name="Base"):
super().__init__(name='LeNet5_'+name)
 self.add(Conv2D(6, kernel_size=(5, 5), strides=(1, 1), activation=activation, input_shape=input_shape, padding="same"))
if pooling == 'avg':
self.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
else:
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1), activation=activation, padding='valid'))
if pooling == 'avg':
self.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
else:
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
if dropout > 0:
self.add(( Dropout(rate = dropout)))
 self.add(Flatten())
 self.add(Dense(120, activation=activation))
self.add(Dense(84, activation=activation))
self.add(Dense(10, activation='softmax'))
self.compile(optimizer='adam',
loss=categorical_crossentropy,
metrics=['accuracy'])
 class LeNet5BN(Sequential):
def __init__(self, input_shape=(rows, cols, channels), name="ImprovedBN"):
super().__init__(name='LeNet5_'+name)
 self.add(Conv2D(6, kernel_size=(5, 5), strides=(1, 1), input_shape=input_shape, padding="same"))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1), padding='valid'))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.add(Flatten())
 self.add(Dense(120))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(Dense(84))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(Dense(10, activation='softmax'))
self.compile(optimizer='adam',
loss=categorical_crossentropy,
metrics=['accuracy'] )
import dataclasses
import os
import sys
from pathlib import Path
import numpy as np
import pandas as pd
import pytorch_pfn_extras.training.extensions as E
import torch
from ignite.engine import Events
from pytorch_pfn_extras.training import IgniteExtensionsManager
from sklearn.model_selection import StratifiedKFold
from torch import nn, optim
from torch.utils.data.dataloader import DataLoader<split>
early_stop = EarlyStopping(patience=5, monitor='val_accuracy', restore_best_weights=True)
 model_LeNet5_base.fit(X_train, y=Y_train,
epochs=25, batch_size=128,
validation_data=(X_val, Y_val),
callbacks=[early_stop] )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=flags.seed)
 y = np.array([int(len(d["annotations"])> 0)for d in dataset_dicts])
 split_inds = list(skf.split(dataset_dicts, y))
train_inds, valid_inds = split_inds[flags.target_fold]
train_dataset = VinbigdataTwoClassDataset(
 [dataset_dicts[i] for i in train_inds],
image_transform=Transform(flags.aug_kwargs),
mixup_prob=flags.mixup_prob,
label_smoothing=flags.label_smoothing, 
)
 valid_dataset = VinbigdataTwoClassDataset([dataset_dicts[i] for i in valid_inds])
<choose_model_class>
preds1 = model_LeNet5_base.evaluate(x = X_val, y = Y_val)
 print("Loss = " + str(preds1[0]))
print("Val Accuracy = " + str(preds1[1]))
train_loader = DataLoader(
 train_dataset,
batch_size=flags.batchsize,
num_workers=flags.num_workers,
shuffle=True,
pin_memory=True, 
)
 valid_loader = DataLoader(
 valid_dataset,
batch_size=flags.valid_batchsize,
num_workers=flags.num_workers,
shuffle=False,
pin_memory=True, 
)
 device = torch.device(flags.device)
 predictor = build_predictor(model_name=flags.model_name, model_mode=flags.model_mode)
 classifier = Classifier(predictor)
 model = classifier
optimizer = optim.Adam([param for param in model.parameters() if param.requires_grad], lr=1e-3)
 trainer = create_trainer(model, optimizer, device)
 ema = EMA(predictor, decay=flags.ema_decay)
 def eval_func(*batch):
loss, metrics = model(*[elem.to(device)for elem in batch])
 if flags.ema_decay > 0:
classifier.prefix = "ema_"
ema.assign()
loss, metrics = model(*[elem.to(device)for elem in batch])
 ema.resume()
classifier.prefix = ""
valid_evaluator = E.Evaluator(
 valid_loader, model, progress_bar=False, eval_func=eval_func, device=device 
)
 log_trigger =(1, "epoch")
 log_report = E.LogReport(trigger=log_trigger)
 extensions = [
log_report,
E.ProgressBarNotebook(update_interval=10 if debug else 100),
E.PrintReportNotebook() ,
E.FailOnNonNumber() ,
]
epoch = flags.epoch
models = {"main": model}
optimizers = {"main": optimizer}
manager = IgniteExtensionsManager(
 trainer, models, optimizers, epoch, extensions=extensions, out_dir=str(outdir), 
)
 manager.extend(valid_evaluator)
 manager.extend(
 E.snapshot_object(predictor, "predictor.pt"), trigger=(flags.snapshot_freq, "epoch")
)
 if flags.scheduler_type != "":
scheduler_type = flags.scheduler_type
print(f"using {scheduler_type} scheduler with kwargs {flags.scheduler_kwargs}")
 manager.extend(
 LRScheduler(optimizer, scheduler_type, flags.scheduler_kwargs),
trigger=flags.scheduler_trigger, 
)
 manager.extend(E.observe_lr(optimizer=optimizer), trigger=log_trigger)
 if flags.ema_decay > 0:
manager.extend(lambda manager: ema() , trigger=(1, "iteration"))
def save_ema_model(manager):
ema.assign()
torch.save(predictor.state_dict() , outdir / "predictor_ema.pt")
 ema.resume()
manager.extend(save_ema_model, trigger=(flags.snapshot_freq, "epoch"))
_ = trainer.run(train_loader, max_epochs=epoch )<save_to_csv>
model_LeNet5 = LeNet5BN(name="Improved_BN")
 model_LeNet5.summary()
torch.save(predictor.state_dict() , outdir / "predictor_last.pt")
 df = log_report.to_dataframe()
df.to_csv(outdir / "log.csv", index=False)
 df<save_to_csv>
early_stop = EarlyStopping(patience=5, monitor='val_accuracy', restore_best_weights=True)
 model_LeNet5.fit(X_train, y=Y_train,
epochs=25, batch_size=128,
validation_data=(X_val, Y_val),
callbacks=[early_stop] )
print("Training done! Start prediction...")
 valid_pred = classifier.predict_proba(valid_loader ).cpu().numpy()
valid_pred_df = pd.DataFrame({
"image_id": [dataset_dicts[i]["image_id"] for i in valid_inds],
"class0": valid_pred[:, 0],
"class1": valid_pred[:, 1]
})
 valid_pred_df.to_csv(outdir/"valid_pred.csv", index=False)
 test_meta = pd.read_csv(inputdir / "vinbigdata-testmeta" / "test_meta.csv")
 dataset_dicts_test = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
 test_dataset = VinbigdataTwoClassDataset(dataset_dicts_test, train=False)
 test_loader = DataLoader(
 test_dataset,
batch_size=flags.valid_batchsize,
num_workers=flags.num_workers,
shuffle=False,
pin_memory=True, 
)
 test_pred = classifier.predict_proba(test_loader ).cpu().numpy()
test_pred_df = pd.DataFrame({
"image_id": [d["image_id"] for d in dataset_dicts_test],
"class0": test_pred[:, 0],
"class1": test_pred[:, 1]
})
 test_pred_df.to_csv(outdir/"test_pred.csv", index=False )<load_from_csv>
preds2 = model_LeNet5.evaluate(x = X_val, y = Y_val)
 print("Loss = " + str(preds2[0]))
print("Val Accuracy = " + str(preds2[1]))
pred_2class = pd.read_csv(inputdir/"vinbigdata2classpred/test_pred.csv")
 low_threshold = 0.0
high_threshold = 0.976
pred_2class<load_from_csv>
conf_mat, class_report, accuracy, faults = analyze_model(model_LeNet5, X_val, Y_val )
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(inputdir/"vinbigdata-detectron2-prediction/results/20210125_all_alb_aug_512_cos/submission.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str(outdir / "submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )<import_modules>
show_faults(faults )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go<load_from_csv>
datagen = ImageDataGenerator(rotation_range=20,
width_shift_range=0.1,
shear_range=0.2,
zoom_range=0.1,
data_format="channels_last")
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 pred_2class<init_hyperparams>
X_train_aug = X_train
Y_train_aug = Y_train
print("Original train size: " , X_train_aug.shape)
 target_size = 32200
batch_size = 200
digits_gen = datagen.flow(X_train, Y_train, batch_size=batch_size)
 for i in range(target_size//batch_size):
batch = digits_gen.next()
X_train_aug = np.append(X_train_aug, batch[0], axis=0)
 Y_train_aug = np.append(Y_train_aug, batch[1], axis=0)
 print("Augmented train size: " , X_train_aug.shape)
low_threshold = 0.005
high_threshold = 0.95<create_dataframe>
model_LeNet5_aug = LeNet5BN(name="Augmented_BN" )
commits_df = pd.DataFrame(columns = ['n_commit', 'low', 'high', 'LB_score'] )<feature_engineering>
early_stop = EarlyStopping(patience=5, monitor='loss', restore_best_weights=True)
 batch_size = 128
model_LeNet5_aug.fit(X_train_aug, Y_train_aug, batch_size=batch_size,
epochs=100,
validation_data=(X_val, Y_val),
callbacks=[early_stop])
n=0
commits_df.loc[n, 'n_commit'] = 0
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.87
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
preds3 = model_LeNet5_aug.evaluate(x = X_val, y = Y_val)
 print("Loss = " + str(preds3[0]))
print("Val Accuracy = " + str(preds3[1]))
n=1
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.90
commits_df.loc[n, 'LB_score'] = 0.244
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
conf_mat, class_report, accuracy, faults = analyze_model(model_LeNet5_aug, X_val, Y_val )
n=2
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.94
commits_df.loc[n, 'LB_score'] = 0.241
commits_df.loc[n, 'LB_private'] = 0.228<feature_engineering>
y_pred = model_LeNet5_aug.predict(X_train_aug,batch_size=None, verbose=0)
 mask = np.argmax(Y_train_aug, axis=1)== np.argmax(y_pred, axis=1)
 x_hit = X_train_aug[mask]
y_hit = Y_train_aug[mask]
x_miss = X_train_aug[~mask]
y_miss = Y_train_aug[~mask]
num_misses = y_miss.shape[0]
sample_idx = sample(range(y_hit.shape[0]), y_miss.shape[0])
 x_hit_reduced = x_hit[sample_idx]
y_hit_reduced = y_hit[sample_idx]
X_train2 = np.append(x_miss,x_hit_reduced, axis=0)
 Y_train2 = np.append(y_miss,y_hit_reduced, axis=0)
 idx_shuffle = list(range(X_train2.shape[0]))
np.random.shuffle(idx_shuffle)
 X_train2 = X_train2[idx_shuffle]
Y_train2 = Y_train2[idx_shuffle]
n=3
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.0
commits_df.loc[n, 'high'] = 0.91
commits_df.loc[n, 'LB_score'] = 0.243
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
model_LeNet5_2 = LeNet5BN(name="Net2" )
n=4
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.002
commits_df.loc[n, 'high'] = 0.9
commits_df.loc[n, 'LB_score'] = 0.244
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
early_stop = EarlyStopping(patience=5, monitor='loss', restore_best_weights=True)
 batch_size = 128
model_LeNet5_2.fit(X_train2, Y_train2,
epochs=100,
callbacks=[early_stop])
n=5
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.005
commits_df.loc[n, 'high'] = 0.95
commits_df.loc[n, 'LB_score'] = 0.241
commits_df.loc[n, 'LB_private'] = 0.228<feature_engineering>
y_pred1 = model_LeNet5_aug.predict(X_train_aug, batch_size=None, verbose=0)
 y_pred2 = model_LeNet5_2.predict(X_train_aug, batch_size=None, verbose=0)
 mask = np.argmax(y_pred1, axis=1)== np.argmax(y_pred2, axis=1)
 X_train3 = X_train_aug[~mask]
Y_train3 = Y_train_aug[~mask]
n=6
commits_df.loc[n, 'n_commit'] = n+1
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.9
commits_df.loc[n, 'LB_score'] = 0.244
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
model_LeNet5_3 = LeNet5BN(name="Net3" )
n=7
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
early_stop = EarlyStopping(patience=5, monitor='loss', restore_best_weights=True)
 batch_size = 128
model_LeNet5_3.fit(X_train3, Y_train3,
epochs=100,
callbacks=[early_stop] )
n=8
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.86
commits_df.loc[n, 'LB_score'] = 0.245
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
def boosted_lenet5(x):
pred1 = model_LeNet5_aug.predict(x,batch_size=None, verbose=0)
 pred2 = model_LeNet5_2.predict(x,batch_size=None, verbose=0)
 pred3 = model_LeNet5_3.predict(x,batch_size=None, verbose=0)
 final_pred = np.argmax(pred1+pred2+pred3, axis = 1)
 return final_pred
n=9
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.875
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.225<feature_engineering>
final_pred = boosted_lenet5(X_val )
n=10
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.0
commits_df.loc[n, 'high'] = 0.875
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.225<feature_engineering>
conf_mat, class_report, accuracy, faults = analyze_results("boosted LeNet5", final_pred, np.argmax(Y_val, axis=1), X_val)
n=11
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.002
commits_df.loc[n, 'high'] = 0.885
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
show_faults(faults )
n=12
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.003
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
acc = [preds1[1], preds2[1], preds3[1], accuracy]
models = ['LeNet5 Base', 'Improved', 'Augmented', 'Boosted LeNet5']
tbl = {"Model": models, "Accuracies": acc}
tbl_df = pd.DataFrame(tbl)
 tbl_df
n=13
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.005
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226<feature_engineering>
test_label = boosted_lenet5(test)
 result = {"ImageId":range(1,1+test_label.shape[0]), "Label":test_label}
submission = pd.DataFrame(result)
 submission.to_csv('submission.csv', index=False )