Kaggle competition: Digit Recognizer
label_df = label_df.sort_values(by='image_id', ascending=1)
 pred_df = pred_df.sort_values(by='image_id', ascending=1)
 ids, labels = label_df.image_id.values, label_df.label.values
preds = np.array([literal_eval(pred)if isinstance(pred, str)else pred for pred in pred_df.label.values])
 print(f'total {len(ids)} images')
 print(f'prediction shape: {preds.shape}, label shape: {labels.shape}' )<define_variables>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
s = labels
psx = preds
K = len(np.unique(s))
thresholds = [np.mean(psx[:,k][s == k])for k in range(K)]
thresholds = np.asarray(thresholds)
 confident_joint = np.zeros(( K, K), dtype = int)
 for i, row in enumerate(psx):
s_label = s[i]
confident_bins = row >= thresholds - 1e-6
num_confident_bins = sum(confident_bins)
 if num_confident_bins == 1:
confident_joint[s_label][np.argmax(confident_bins)] += 1
elif num_confident_bins > 1:
confident_joint[s_label][np.argmax(row)] += 1
confident_joint = cleanlab.latent_estimation.calibrate_confident_joint(
 confident_joint, s)
 cleanlab.util.print_joint_matrix(confident_joint)
 MIN_NUM_PER_CLASS = 5
prune_count_matrix = cleanlab.pruning.keep_at_least_n_per_class(
 prune_count_matrix=confident_joint.T,
n=MIN_NUM_PER_CLASS, 
)
 s_counts = np.bincount(s)
 noise_masks_per_class = []
for k in range(K):
noise_mask = np.zeros(len(psx), dtype=bool)
 psx_k = psx[:, k]
if s_counts[k] > MIN_NUM_PER_CLASS:
for j in range(K):
if k != j:
num2prune = prune_count_matrix[k][j]
if num2prune > 0:
margin = psx_k - psx[:, j]
s_filter = s == j
threshold = -np.partition(
 -margin[s_filter], num2prune - 1 
)[num2prune - 1]
noise_mask = noise_mask |(s_filter &(margin >= threshold))
noise_masks_per_class.append(noise_mask)
 else:
noise_masks_per_class.append(np.zeros(len(s), dtype=bool))
label_errors_bool = np.stack(noise_masks_per_class ).any(axis=0)
 for i, pred_label in enumerate(psx.argmax(axis=1)) :
if label_errors_bool[i] and np.all(pred_label == s[i]):
label_errors_bool[i] = False
label_errors_idx = np.arange(len(s)) [label_errors_bool]
self_confidence = np.array(
 [np.mean(psx[i][s[i]])for i in label_errors_idx] 
)
 margin = self_confidence - psx[label_errors_bool].max(axis=1)
 label_errors_idx = label_errors_idx[np.argsort(margin)]<feature_engineering>
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train /= 255
X_test /= 255
total_idx = np.arange(len(ids))
clean_idx = np.array([idx for idx in total_idx if idx not in label_errors_idx])
 guesses = np.stack(noise_masks_per_class ).argmax(axis=0)
 guesses[clean_idx] = labels[clean_idx]
clean_ids = ids[clean_idx]
clean_labels = labels[clean_idx]
clean_guesses = guesses[clean_idx]
noisy_ids = ids[label_errors_idx]
noisy_labels = labels[label_errors_idx]
noisy_guesses = guesses[label_errors_idx]
print(f'[clean ratio] \t {len(clean_idx)/ len(total_idx)* 100:.2f}%')
 print(f'[noise ratio] \t {len(noisy_ids)/ len(total_idx)* 100:.2f}%' )<prepare_output>
y_train = train.pop("label")
 y_train = y_train.values
print(y_train.shape)
 y_train = to_categorical(y_train, num_classes=10)
 print(y_train.shape)
 print(y_train[0] )
all_data = pd.DataFrame({'image_id': ids,
'given_label': labels,
'guess_label': guesses})
 all_data['is_noisy'] =(all_data.given_label != all_data.guess_label)
 all_data['max_prob'] = preds.max(axis=1 )<define_variables>
import tensorflow as tf
class_colors = np.array(['
num2class = [f'{idx}-{elem}' for idx, elem in enumerate(num2class)]<install_modules>
def squeeze_excite_block(filters,input):
se = tf.keras.layers.GlobalAveragePooling2D()(input)
 se = tf.keras.layers.Reshape(( 1, filters))(se)
 se = tf.keras.layers.Dense(filters//16, activation='relu' )(se)
 se = tf.keras.layers.Dense(filters, activation='sigmoid' )(se)
 se = tf.keras.layers.multiply([input, se])
 return se
!pip install -U -q.. /input/resnest/resnest-0.0.5-py3-none-any.whl<import_modules>
def make_model() :
s = tf.keras.Input(shape=X_train.shape[1:])
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(s)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = squeeze_excite_block(32,x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = squeeze_excite_block(32,x)
 x = tf.keras.layers.AveragePooling2D(2 )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = squeeze_excite_block(32,x)
 x = tf.keras.layers.AveragePooling2D(2 )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.Conv2D(32,(3,3),activation='relu',padding='same' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = squeeze_excite_block(32,x)
 x = tf.keras.layers.AveragePooling2D(2 )(x)
 x = tf.keras.layers.concatenate([tf.keras.layers.GlobalMaxPooling2D()(x),
tf.keras.layers.GlobalAveragePooling2D()(x)])
 x = tf.keras.layers.Dense(10,activation='softmax',use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.00025))(x)
 return tf.keras.Model(inputs=s, outputs=x )
HorizontalFlip, VerticalFlip, Transpose, RandomResizedCrop, Compose, Normalize, ShiftScaleRotate, CenterCrop, Resize, RandomResizedCrop 
)
<import_modules>
model=make_model()
model.compile(optimizer=optimizers.Adam(lr=0.0001), loss='categorical_crossentropy',metrics=['accuracy'])
 model.summary()
import timm<init_hyperparams>
model.fit(x=X_train, y=y_train, batch_size=32, epochs=15 )
CFG = {
'seed': 1337,
'img_size': 512,
'bs': 32,
'num_workers': 4,
'tta': 4,
}<load_from_csv>
pred = model.predict(X_test,verbose=1)
 predictions = pred.argmax(axis=-1 )
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<set_options>
sub = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 submission = sub['ImageId']
def seed_everything(seed: int):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False<normalization>
pred = pd.DataFrame(data=predictions ,columns=["Label"])
 DT = pd.merge(submission , pred, on=None, left_index= True,
right_index=True)
 DT.head()
def get_img(path):
return cv2.imread(path)[:, :, ::-1]<define_variables>
DT.to_csv('submission.csv',index = False )