Kaggle competition: Digit Recognizer
seed_everything(CFG['seed'] )<categorify>
from tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPool2D, BatchNormalization, AvgPool2D
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.utils import to_categorical
from sklearn.metrics import confusion_matrix
from tensorflow.random import set_seed
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
class CassavaDataset(Dataset):
def __init__(self, df, data_root, transforms=None):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
img = get_img(f"{self.data_root}/{self.df.iloc[index]['image_id']}")
 if self.transforms:
img = self.transforms(image=img)['image']
return img<feature_engineering>
df_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
class CassvaClassifierV1(nn.Module):
def __init__(self, n_classes: int = 5, dropout: float =.5):
super().__init__()
self.backbone = resnest50_fast_4s2x40d(pretrained=False)
 self.pool = nn.AdaptiveAvgPool2d(1)
 self.dropout = nn.Dropout(p=dropout)
 self.emb_size: int = 2048
self.classifier = nn.Linear(self.emb_size, n_classes)
 def cnn_feature_extractor(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x1 = self.backbone.layer1(x)
 x2 = self.backbone.layer2(x1)
 x3 = self.backbone.layer3(x2)
 x4 = self.backbone.layer4(x3)
 return x4
def forward(self, x):
x = self.cnn_feature_extractor(x)
 x = self.pool(x)
 x = self.dropout(x)
 x = x.view(-1, self.emb_size)
 x = self.classifier(x)
 return x<choose_model_class>
np.random.seed(42)
 set_seed(42 )
class CassvaClassifierV2(nn.Module):
def __init__(self, n_classes: int = 5):
super().__init__()
self.model = timm.create_model('tf_efficientnet_b4_ns', pretrained=False)
 self.model.classifier = nn.Linear(self.model.classifier.in_features, n_classes)
 def forward(self, x):
return self.model(x )<choose_model_class>
def doSubmission(y_pred):
test_Id = np.arange(1, y_pred.size+1, dtype=np.int)
 pred_dict = {"ImageId": test_Id, "Label": y_pred}
df = pd.DataFrame(pred_dict)
 df.to_csv("sample_submission.csv", index=False, index_label=False )
class CassvaClassifierV3(nn.Module):
def __init__(self, n_classes: int = 5):
super().__init__()
self.model = timm.create_model('tf_efficientnet_b3_ns', pretrained=False)
 self.model.classifier = nn.Linear(self.model.classifier.in_features, n_classes)
 def forward(self, x):
return self.model(x )<predict_on_test>
y = df_train.label.to_numpy()
X = df_train.drop(columns=["label"] ).to_numpy()
X = X / 255.0
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
for imgs in data_loader:
image_preds = model(imgs.to(device ).float())
 image_preds_all += [torch.softmax(image_preds, 1 ).cpu().numpy() ]
return np.concatenate(image_preds_all, axis=0 )<set_options>
X_totrain = X.reshape(X.shape[0], 28, 28, 1 )
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y)
 X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28, 1 )
def get_models(model_paths, recipe: str):
models = []
for model_path in model_paths:
model_name: str = model_path.split('/')[-1]
n_folds: int = int([x[-1] for x in model_name.split('-')if x.startswith('fold')][0])
 n_epochs: int = int([x[6:] for x in model_name.split('-')if x.startswith('epochs')][0])
 if not model_name.startswith(recipe):
continue
if model_name.startswith('resnest50_fast_4s2x40d-fmix-cutmix-'):
model = CassvaClassifierV1().to(device)
 elif model_name.startswith('resnest50_fast_4s2x40d-cutmix-fmix-'):
model = CassvaClassifierV1().to(device)
 elif model_name.startswith('resnest50_fast_4s2x40d-fcl-'):
model = CassvaClassifierV1().to(device)
 elif model_name.startswith('effnetb4-fcl-'):
model = CassvaClassifierV2().to(device)
 elif model_name.startswith('effnetb3-cutmix-scce-'):
model = CassvaClassifierV3().to(device)
 elif model_name.startswith('effnetb4-cutmix-fmix-'):
model = CassvaClassifierV2().to(device)
 elif model_name.startswith('effnetb4-pseudo-cutmix-fmix-'):
model = CassvaClassifierV2().to(device)
 else:
continue
print(f'[+] load {model_name}')
 model.load_state_dict(torch.load(model_path, map_location=device))
model.eval()
models.append(model)
 return models<create_dataframe>
y_cat = to_categorical(y, 10)
 y_train_cat = to_categorical(y_train, 10)
 y_test_cat = to_categorical(y_test, 10 )
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms())
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True
)<define_variables>
test = df_test.to_numpy(np.float64)
 test = test.reshape(test.shape[0], 28, 28, 1)
 test /= 255.0
recipe_list = [
'effnetb4-cutmix-fmix-',
'effnetb4-fcl-',
'resnest50_fast_4s2x40d-cutmix-fmix-',
'resnest50_fast_4s2x40d-fcl-',
'resnest50_fast_4s2x40d-fmix-cutmix-',
]<feature_engineering>
def convNeuralNetwork() :
cnn = Sequential()
cnn.add(Conv2D(filters=6, kernel_size=(5, 5), strides=(1, 1),
input_shape=(28, 28, 1), activation="tanh", padding="same"))
cnn.add(BatchNormalization())
 cnn.add(AvgPool2D(pool_size=(2, 2), strides=(1, 1), padding="same"))
cnn.add(Dropout(0.2))
cnn.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1),
activation="tanh", padding="same"))
cnn.add(BatchNormalization())
 cnn.add(AvgPool2D(pool_size=(2, 2), strides=(2, 2), padding="same"))
cnn.add(Dropout(0.2))
cnn.add(Conv2D(120, kernel_size=(5, 5), strides=(1, 1),
activation="tanh", padding="same"))
cnn.add(AvgPool2D(pool_size=(2, 2), strides=(2, 2), padding="same"))
cnn.add(Dropout(0.2))
cnn.add(Flatten())
 cnn.add(Dense(units=84, activation="tanh"))
cnn.add(Dropout(0.2))
cnn.add(Dense(units=10, activation="softmax"))
cnn.compile(optimizer="SGD", loss="categorical_crossentropy",
metrics=["accuracy"])
 return cnn
preds = []
with torch.no_grad() :
for recipe in recipe_list:
models = get_models(model_paths, recipe)
 preds_per_recipe = np.mean(
 [
np.mean([inference_one_epoch(model, tst_loader, device)for _ in range(CFG['tta'])], axis=0)
 for model in models
], axis=0 
)
 preds.append(preds_per_recipe)
 del models<define_variables>
early_stopping = EarlyStopping(monitor="val_loss", patience=4, verbose=1,
restore_best_weights=True)
 lr_reducer = ReduceLROnPlateau(monitor="val_loss", factor=0.1, patience=3,
verbose=1)
 cnn = convNeuralNetwork()
cnn_hist = cnn.fit(X_train, y_train_cat, validation_data=(X_test, y_test_cat),
epochs=100, batch_size=16, callbacks=[early_stopping, lr_reducer] )
tst_preds = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4] 
<feature_engineering>
early_stopping = EarlyStopping(monitor="loss", patience=4, restore_best_weights=True, verbose=1)
 model_checkpoint = ModelCheckpoint(filepath="./", monitor="loss", verbose=1,
save_best_only=True, save_weights_only=True)
 lr_reducer = ReduceLROnPlateau(monitor="loss", factor=0.1, patience=3,
verbose=1)
 model= convNeuralNetwork()
model_hist = model.fit(X_totrain, y_cat, epochs=100, batch_size=16,
callbacks=[early_stopping, lr_reducer, model_checkpoint] )
<save_to_csv><EOS> 
y_pred = model.predict(test ).argmax(1)
 doSubmission(y_pred )