Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_search_space>
from matplotlib import pyplot as plt
import math, os, re, time, random
import numpy as np, pandas as pd, seaborn as sns
import tensorflow as tf
from sklearn.model_selection import train_test_split
eff_b0_cfg_s =
eff_b0_cfg = OmegaConf.create(eff_b0_cfg_s )<define_variables>
rank_0_tensor = tf.constant(1)
 print(rank_0_tensor); print('')
 rank_1_tensor = tf.constant([1, 0, 0])
 print(rank_1_tensor); print('')
 rank_2_tensor = tf.constant([[1, 0, 0],
[0, 1, 0],
[0, 0, 1]])
 print(rank_2_tensor )
name = '14-10-36'
cfg = eff_b0_cfg
do_predict = True
do_submit = True
img_dir = '.. /input/cassava-leaf-disease-merged/train/'
label_path = '.. /input/cassava-leaf-disease-merged/merged.csv'
log_dir = os.path.join('.. /input/cassava-public-ckpt', name)
 n_folds = len(glob(os.path.join(log_dir, 'checkpoints/*.ckpt')))
 num2class = ["Cassava Bacterial Blight(CBB)", "Cassava Brown Streak Disease(CBSD)", "Cassava Green Mottle(CGM)", "Cassava Mosaic Disease(CMD)", "Healthy"]<feature_engineering>
rank_0_tensor = tf.constant(1, dtype = tf.float16)
 print(rank_0_tensor); print('')
 rank_1_tensor = tf.constant([1, 0, 0], dtype = tf.float32)
 print(rank_1_tensor); print('')
 rank_2_tensor = tf.constant([[1, 0, 0],
[0, 1, 0],
[0, 0, 1]], dtype = tf.int32)
 print(rank_2_tensor )
seed_everything(42)
 label_df = pd.read_csv(label_path)
 if 'fold' not in label_df.columns:
skf = StratifiedKFold(n_splits=5, shuffle=True)
 label_df.loc[:, 'fold'] = 0
for fold_num,(train_index, val_index)in enumerate(skf.split(X=label_df.index, y=label_df.label.values)) :
label_df.loc[label_df.iloc[val_index].index, 'fold'] = fold_num
if do_predict:
infer = pl.Trainer(gpus=1)
 oof_dict = {'image_id': [], 'label': [], 'fold': []}
for fold_num in range(n_folds):
val_df = label_df[label_df.fold == fold_num]
test_dataset = TestDataset(img_dir, val_df, img_size=cfg.img_size)
 test_dataloader = DataLoader(test_dataset,
batch_size=cfg.batch_size,
num_workers=4,
shuffle=False)
 state_dict = get_state_dict_from_checkpoint(log_dir, fold_num)
 model = LitTester(cfg.network, state_dict)
 pred = infer.test(model, test_dataloaders=test_dataloader, verbose=False)[0]
oof_dict['image_id'].extend(val_df.image_id.values)
 oof_dict['label'].extend(pred['prob'].tolist())
 oof_dict['fold'].extend([fold_num] * len(pred['prob']))
pred_df = pd.DataFrame(oof_dict)
 pred_df.to_csv('oof.csv', index=False)
 else:
pred_df = pd.read_csv(os.path.join(log_dir, 'oof.csv'))<load_from_csv>
print(type(rank_2_tensor.numpy()))
print(rank_2_tensor.numpy()); print('')
 tensor_to_array = np.add(rank_2_tensor, 1)
 print(type(tensor_to_array))
print(tensor_to_array); print('')
 array_to_tensor = tf.add(rank_2_tensor.numpy() , 1)
 print(array_to_tensor )
if do_submit:
sub = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 infer = pl.Trainer(gpus=1)
 test_dataset = TestDataset('.. /input/cassava-leaf-disease-classification/test_images',
sub,
img_size=cfg.img_size)
 test_dataloader = DataLoader(test_dataset,
batch_size=cfg.batch_size,
num_workers=4,
shuffle=False)
 preds = []
for fold_num in range(n_folds):
state_dict = get_state_dict_from_checkpoint(log_dir, fold_num)
 model = LitTester(cfg.network, state_dict)
 pred = infer.test(model, test_dataloaders=test_dataloader, verbose=False)[0]
preds.append(pred['prob'])
 sub['label'] = np.argmax(np.mean(preds, axis=0), axis=1)
 sub.to_csv(os.path.join(os.getcwd() , 'submission.csv'), index=False )<sort_values>
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(256, activation='relu', input_shape =(784,)))
 model.add(tf.keras.layers.Dense(128, activation='swish'))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["categorical_accuracy"])
 model.summary()
label_df = label_df.sort_values(by='image_id', ascending=1)
 pred_df = pred_df.sort_values(by='image_id', ascending=1)
 ids, labels = label_df.image_id.values, label_df.label.values
preds = np.array([literal_eval(pred)if isinstance(pred, str)else pred for pred in pred_df.label.values])
 print(f'total {len(ids)} images')
 print(f'prediction shape: {preds.shape}, label shape: {labels.shape}' )<define_variables>
model = tf.keras.models.Sequential([
tf.keras.layers.Dense(256, activation = 'relu', input_shape =(784,)) ,
tf.keras.layers.Dense(128, activation = 'swish'),
tf.keras.layers.Dense(10, activation = 'softmax')
 ])
 model.compile(optimizer = "adam", loss = "categorical_crossentropy", metrics = ["categorical_accuracy"])
 model.summary()
s = labels
psx = preds
K = len(np.unique(s))
thresholds = [np.mean(psx[:,k][s == k])for k in range(K)]
thresholds = np.asarray(thresholds)
 confident_joint = np.zeros(( K, K), dtype = int)
 for i, row in enumerate(psx):
s_label = s[i]
confident_bins = row >= thresholds - 1e-6
num_confident_bins = sum(confident_bins)
 if num_confident_bins == 1:
confident_joint[s_label][np.argmax(confident_bins)] += 1
elif num_confident_bins > 1:
confident_joint[s_label][np.argmax(row)] += 1
confident_joint = cleanlab.latent_estimation.calibrate_confident_joint(
 confident_joint, s)
 cleanlab.util.print_joint_matrix(confident_joint)
 MIN_NUM_PER_CLASS = 5
prune_count_matrix = cleanlab.pruning.keep_at_least_n_per_class(
 prune_count_matrix=confident_joint.T,
n=MIN_NUM_PER_CLASS, 
)
 s_counts = np.bincount(s)
 noise_masks_per_class = []
for k in range(K):
noise_mask = np.zeros(len(psx), dtype=bool)
 psx_k = psx[:, k]
if s_counts[k] > MIN_NUM_PER_CLASS:
for j in range(K):
if k != j:
num2prune = prune_count_matrix[k][j]
if num2prune > 0:
margin = psx_k - psx[:, j]
s_filter = s == j
threshold = -np.partition(
 -margin[s_filter], num2prune - 1 
)[num2prune - 1]
noise_mask = noise_mask |(s_filter &(margin >= threshold))
noise_masks_per_class.append(noise_mask)
 else:
noise_masks_per_class.append(np.zeros(len(s), dtype=bool))
label_errors_bool = np.stack(noise_masks_per_class ).any(axis=0)
 for i, pred_label in enumerate(psx.argmax(axis=1)) :
if label_errors_bool[i] and np.all(pred_label == s[i]):
label_errors_bool[i] = False
label_errors_idx = np.arange(len(s)) [label_errors_bool]
self_confidence = np.array(
 [np.mean(psx[i][s[i]])for i in label_errors_idx] 
)
 margin = self_confidence - psx[label_errors_bool].max(axis=1)
 label_errors_idx = label_errors_idx[np.argsort(margin)]<feature_engineering>
inputs = tf.keras.Input(shape =(784,))
x = tf.keras.layers.Dense(256, activation = 'relu' )(inputs)
 x = tf.keras.layers.Dense(128, activation = 'swish' )(x)
 outputs = tf.keras.layers.Dense(10, activation = 'softmax' )(x)
 model = tf.keras.Model(inputs = inputs, outputs = outputs)
 model.compile(optimizer = "adam", loss = "categorical_crossentropy", metrics = ["categorical_accuracy"])
 model.summary()
total_idx = np.arange(len(ids))
clean_idx = np.array([idx for idx in total_idx if idx not in label_errors_idx])
 guesses = np.stack(noise_masks_per_class ).argmax(axis=0)
 guesses[clean_idx] = labels[clean_idx]
clean_ids = ids[clean_idx]
clean_labels = labels[clean_idx]
clean_guesses = guesses[clean_idx]
noisy_ids = ids[label_errors_idx]
noisy_labels = labels[label_errors_idx]
noisy_guesses = guesses[label_errors_idx]
print(f'[clean ratio] \t {len(clean_idx)/ len(total_idx)* 100:.2f}%')
 print(f'[noise ratio] \t {len(noisy_ids)/ len(total_idx)* 100:.2f}%' )<prepare_output>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train.head()
all_data = pd.DataFrame({'image_id': ids,
'given_label': labels,
'guess_label': guesses})
 all_data['is_noisy'] =(all_data.given_label != all_data.guess_label)
 all_data['max_prob'] = preds.max(axis=1 )<define_variables>
labels = train['label']
train = train.drop('label', axis = 1)
 train = train / 255.0
test = test / 255.0
class_colors = np.array(['
num2class = [f'{idx}-{elem}' for idx, elem in enumerate(num2class)]<load_pretrained>
labels = tf.one_hot(labels, depth = 10 ).numpy()
with open('.. /input/train-weights-optimization/best_weights.json', 'r')as f:
weights_dict = json.load(f)
 weights_dict<load_pretrained>
datagen = tf.keras.preprocessing.image.ImageDataGenerator(
 rotation_range = 20,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1
)
normal_configs = []
tta_configs = []
normal_model_dirs = []
tta_model_dirs = []
for model_dir in weights_dict.keys() :
assert len(glob.glob(f'{model_dir}/*.yml')) ==1
config_path = glob.glob(f'{model_dir}/*.yml')[0]
with open(config_path)as f:
config = yaml.load(f)
 if 'valid_augmentation' in config['tag'].keys() :
tta_model_dirs.append(model_dir)
 tta_configs.append(config)
 else:
normal_model_dirs.append(model_dir)
 normal_configs.append(config)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'
OUTPUT_DIR = './'<compute_test_metric>
EPOCHS = 45
BATCH_SIZE = 64
NUM_NETS = 25
VERBOSE = 0
def get_score(y_true, y_pred):
return accuracy_score(y_true, y_pred)
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True 
<load_from_csv>
model = [0] * NUM_NETS
for j in range(NUM_NETS):
model[j] = tf.keras.models.Sequential()
model[j].add(tf.keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', input_shape =(28, 28, 1)))
 model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu'))
model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Conv2D(32, kernel_size = 5, strides = 2, padding = 'same', activation = 'relu'))
model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Dropout(0.4))
model[j].add(tf.keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu'))
model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu'))
model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Conv2D(64, kernel_size = 5, strides = 2, padding = 'same', activation = 'relu'))
model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Dropout(0.4))
model[j].add(tf.keras.layers.Conv2D(128, kernel_size = 4, activation = 'relu'))
model[j].add(tf.keras.layers.BatchNormalization())
 model[j].add(tf.keras.layers.Flatten())
 model[j].add(tf.keras.layers.Dropout(0.4))
model[j].add(tf.keras.layers.Dense(10, activation = 'softmax'))
model[j].compile(optimizer = "adam", loss = "categorical_crossentropy", metrics = ["categorical_accuracy"] )
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test.head()<normalization>
lr_callback = tf.keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * NUM_NETS
for j in range(NUM_NETS):
X_train, X_val, y_train, y_val = train_test_split(train, labels, test_size = 0.1)
 STEPS_PER_EPOCH = X_train.shape[0] // 64
history[j] = model[j].fit_generator(datagen.flow(X_train, y_train, batch_size = BATCH_SIZE),
epochs = EPOCHS, steps_per_epoch = STEPS_PER_EPOCH,
validation_data =(X_val, y_val), callbacks = [lr_callback],
verbose = VERBOSE)
 print(f"CNN {j + 1}: Epochs={EPOCHS}, Train accuracy={max(history[j].history['categorical_accuracy'])}, Validation accuracy={max(history[j].history['val_categorical_accuracy'])}" )
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['image_id'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image
class TTADataset(Dataset):
def __init__(self, df, image_path, ttas):
self.df = df
self.file_names = df['image_id'].values
self.labels = df['label'].values
self.image_path = image_path
self.ttas = ttas
def __len__(self)-> int:
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{self.image_path}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 imglist=[tta(image=image)['image'] for tta in self.ttas]
image=torch.stack(imglist)
 label = torch.tensor(self.labels[idx] ).long()
return image, label<normalization>
preds = np.zeros(( test.shape[0],10))
for j in range(NUM_NETS):
preds += model[j].predict(test)/ NUM_NETS
probs = pd.DataFrame(preds)
 probs.to_csv('ensemble_probs')
 probs.columns = probs.columns.astype(str)
 print(probs.columns)
 probs.head()
def _get_augmentations(aug_list, cfg):
process = []
for aug in aug_list:
if aug == 'Resize':
process.append(Resize(cfg['size'], cfg['size']))
elif aug == 'RandomResizedCrop':
process.append(RandomResizedCrop(cfg['size'], cfg['size']))
elif aug == 'CenterCrop':
process.append(CenterCrop(CFG['size'], CFG['size']))
elif aug == 'Transpose':
process.append(Transpose(p=0.5))
elif aug == 'HorizontalFlip':
process.append(HorizontalFlip(p=0.5))
elif aug == 'VerticalFlip':
process.append(VerticalFlip(p=0.5))
elif aug == 'ShiftScaleRotate':
process.append(ShiftScaleRotate(p=0.5))
elif aug == 'Normalize':
process.append(Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
))
else:
raise ValueError(f'{aug} is not suitable')
 process.append(ToTensorV2())
 return process<categorify>
submission = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 submission['Label'] = preds.argmax(axis = 1)
 submission.to_csv("ensemble.csv", index = False)
 submission.head(10 )
def get_transforms(*, aug_list, cfg):
return Compose(
 _get_augmentations(aug_list, cfg)
 )<choose_model_class>
prev_cnn_probs = pd.read_csv('.. /input/mnistsavedprobs/ensemble_probs')
 prev_cnn_probs = prev_cnn_probs.drop('Unnamed: 0', axis = 1)
 print(prev_cnn_probs.columns)
 prev_cnn_probs.head()
class CustomModel(nn.Module):
def __init__(self, model_name, target_size, pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 if hasattr(self.model, 'classifier'):
n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, target_size)
 elif hasattr(self.model, 'fc'):
n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, target_size)
 elif hasattr(self.model, 'head'):
n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, target_size)
 def forward(self, x):
x = self.model(x)
 return x<categorify>
new_probs = probs.add(prev_cnn_probs ).divide(2)
 new_probs.head()
<load_pretrained><EOS> 
submission2 = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 submission2['Label'] = new_probs.values.argmax(axis = 1)
 submission2.to_csv("ensemble2.csv", index = False)
 submission2.head(10 )