Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
%matplotlib inline
print(torch.__version__ )
def transform(image,label):
DIM = IMAGE_SIZE[0]
DIM1 = IMAGE_SIZE[1]
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
image2, label2 = cutmix(image, label, CUTMIX_PROB)
 image3, label3 = mixup(image, label, MIXUP_PROB)
 imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM1,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4<create_dataframe>
random_seed = 42
torch.backends.cudnn.enabled = False
torch.manual_seed(random_seed)
 train_df = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_df = pd.read_csv(".. /input/digit-recognizer/test.csv")
 y = train_df["label"]
x = train_df.drop("label", axis = 1)
 X_train, X_valid, y_train, y_valid = train_test_split(x, y, test_size=0.15, shuffle=True)
 num_epoch = 25
batch_size_train = 32
batch_size_test = 32
learning_rate = 0.002
momentum = 0.9
log_interval = 100
def load_dataset(filenames, labeled = True, ordered = False , training = True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_original_tfrecord, num_parallel_calls = AUTO)
 if training:
dataset_tp = dataset.filter(filter_tp_tfrecord)
 dataset_fp = dataset.filter(lambda x: not filter_tp_tfrecord(x))
dataset_fp = dataset_fp.map(partial(read_labeled_tfrecord_fp, random_shift=False), num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(partial(read_labeled_tfrecord, random_shift=False), num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(apply_augmentation, num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(preprocess, num_parallel_calls = AUTO)
 dataset_fp = dataset_fp.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset_fp = dataset_fp.map(apply_augmentation, num_parallel_calls = AUTO)
 dataset_fp = dataset_fp.map(preprocess, num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 dataset_tp = dataset_tp.map(transform, num_parallel_calls=AUTO)
 dataset_tp = dataset_tp.unbatch()
dataset_fp = dataset_fp.shuffle(256)
 dataset = dataset_tp.concatenate(dataset_fp)
 dataset = dataset.shuffle(2048 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE)
 else:
dataset_tp = dataset.filter(filter_tp_tfrecord)
 dataset = dataset_tp.map(read_labeled_tfrecord , num_parallel_calls = AUTO)
 dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset = dataset.batch(VAL_BATCH_SIZE ).cache()
dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
class MNISTDataset(Dataset):
def __init__(self, data, target, train=True, transform=None):

 self.train = train
if self.train :
self.data = data
self.labels = np.asarray(target.iloc[:])
 else:
self.data = data
self.labels = None
self.height = 28
self.width = 28
self.transform = transform
def __getitem__(self, index):
img_as_np = np.asarray(self.data.iloc[index][0:] ).reshape(self.height, self.width ).astype('uint8')
 img_as_img = Image.fromarray(img_as_np)
 img_as_img = img_as_img.convert('L')
 img_as_tensor = img_as_img
if self.train:
single_image_label = self.labels[index]
else:
single_image_label = None
if self.transform is not None:
img_as_tensor = self.transform(img_as_img)
 if self.train:
return(img_as_tensor, single_image_label)
 else:
return img_as_tensor
def __len__(self):
return len(self.data.index )
TP_FILENAMES = FILENAMES[:8]
FP_FILENAMES = FILENAMES[8:]<create_dataframe>
def calculate_img_stats_full(dataset):
imgs_ = torch.stack([img for img,_ in dataset],dim=1)
 imgs_ = imgs_.view(1,-1)
 imgs_mean = imgs_.mean(dim=1)
 imgs_std = imgs_.std(dim=1)
 return imgs_mean,imgs_std
def get_dataset_(filenames, fp_filenames = None, training = True):
if training:
dataset = load_dataset(filenames , training = True)
 dataset = dataset.shuffle(256)
 dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.unbatch()
if fp_filenames is not None:
dataset_fp = load_dataset(fp_filenames, training = True, FP=True)
 dataset_fp = dataset_fp.shuffle(256)
 dataset = dataset.concatenate(dataset_fp)
 dataset = dataset.shuffle(2048 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 else:
dataset = load_dataset(filenames , training = False)
 dataset = dataset.batch(VAL_BATCH_SIZE ).cache()
dataset = dataset.prefetch(AUTO)
 return dataset<groupby>
transformations_train = transforms.Compose([transforms.RandomRotation(15),
transforms.RandomAffine(0, shear=10, scale=(0.8,1.2)) ,
transforms.ToTensor() ,
transforms.Normalize(mean=[0.1310], std=[0.3085])
 ])
 transformations_valid = transforms.Compose([transforms.ToTensor() ,
transforms.Normalize(mean=[0.1310], std=[0.3085])
 ])
 train = MNISTDataset(X_train, y_train, True, transformations_train)
 valid = MNISTDataset(X_valid, y_valid, True, transformations_valid)
 test = MNISTDataset(data=test_df, target=None, train=False, transform=transformations_valid )
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
y_true = tf.reshape(y_true, tf.shape(y_pred))
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )<normalization>
train_loader = DataLoader(train, batch_size=batch_size_train,num_workers=2, shuffle=True)
 valid_loader = DataLoader(valid, batch_size=batch_size_test, num_workers=2, shuffle=True)
 test_loader = DataLoader(test, batch_size=batch_size_test, shuffle=False )
def cosine_decay_with_warmup(global_step,
learning_rate_base,
total_steps,
warmup_learning_rate=0.0,
warmup_steps= 0,
hold_base_rate_steps=0):
if total_steps < warmup_steps:
raise ValueError('total_steps must be larger or equal to '
'warmup_steps.')
 learning_rate = 0.5 * learning_rate_base *(1 + tf.cos(
 np.pi * 
(tf.cast(global_step, tf.float32)- warmup_steps - hold_base_rate_steps 
)/ float(total_steps - warmup_steps - hold_base_rate_steps)))
 if hold_base_rate_steps > 0:
learning_rate = tf.where(
 global_step > warmup_steps + hold_base_rate_steps,
learning_rate, learning_rate_base)
 if warmup_steps > 0:
if learning_rate_base < warmup_learning_rate:
raise ValueError('learning_rate_base must be larger or equal to '
'warmup_learning_rate.')
 slope =(learning_rate_base - warmup_learning_rate)/ warmup_steps
warmup_rate = slope * tf.cast(global_step,
tf.float32)+ warmup_learning_rate
learning_rate = tf.where(global_step < warmup_steps, warmup_rate,
learning_rate)
 return tf.where(global_step > total_steps, 0.0, learning_rate,
name='learning_rate')
 rng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH)) ]
WARMUP_STEPS = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 y = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS)for x in rng]
sns.set(style='whitegrid')
 fig, ax = plt.subplots(figsize=(20, 6))
plt.plot(rng, y )<choose_model_class>
class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.conv_block = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2)
)
 self.linear_block = nn.Sequential(
 nn.Dropout(p=0.5),
nn.Linear(128*7*7, 128),
nn.BatchNorm1d(128),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(128, 64),
nn.BatchNorm1d(64),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(64, 10)
)
 def forward(self, x):
x = self.conv_block(x)
 x = x.view(x.size(0), -1)
 x = self.linear_block(x)
 return x
class WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):
def __init__(self,
learning_rate_base,
total_steps,
global_step_init=0,
warmup_learning_rate=0.0,
warmup_steps=0,
hold_base_rate_steps=0,
verbose=0):
super(WarmUpCosineDecayScheduler, self ).__init__()
self.learning_rate_base = learning_rate_base
self.total_steps = total_steps
self.global_step = global_step_init
self.warmup_learning_rate = warmup_learning_rate
self.warmup_steps = warmup_steps
self.hold_base_rate_steps = hold_base_rate_steps
self.verbose = verbose
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.global_step = self.global_step + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
lr = cosine_decay_with_warmup(global_step=self.global_step,
learning_rate_base=self.learning_rate_base,
total_steps=self.total_steps,
warmup_learning_rate=self.warmup_learning_rate,
warmup_steps=self.warmup_steps,
hold_base_rate_steps=self.hold_base_rate_steps)
 K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: setting learning '
'rate to %s.' %(self.global_step + 1, lr.numpy()))
total_steps = int(EPOCHS * STEPS_PER_EPOCH)
 warmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 warmup_learning_rate = WARMUP_LEARNING_RATE<choose_model_class>
cnn_model = Net()
criterion = nn.CrossEntropyLoss()
if torch.cuda.is_available() :
cnn_model.cuda()
criterion.cuda()
optimizer = optim.Adam(params=cnn_model.parameters() , lr=learning_rate)
 exp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min')
 train_losses = []
train_counter = []
test_losses = []
test_counter = [i*len(train_loader.dataset)for i in range(1, num_epoch + 1)]
best_model_wts = copy.deepcopy(cnn_model.state_dict())
 best_acc = 0.0
since = time.time()
for epoch in range(1, num_epoch + 1):
cnn_model.train()
for i,(images, labels)in enumerate(train_loader):
images = Variable(images ).cuda()
labels = Variable(labels ).cuda()
optimizer.zero_grad()
outputs = cnn_model(images)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
if(i + 1)% log_interval == 0:
print('Train Epoch: {} [{}/{}({:.0f}%)]\tLoss: {:.6f}'.format(
 epoch,(i + 1)* len(images), len(train_loader.dataset),
100.*(i + 1)/ len(train_loader), loss.data))
train_losses.append(loss.item())
 train_counter.append(( i*64)+(( epoch-1)*len(train_loader.dataset)))
 cnn_model.eval()
loss = 0
running_corrects = 0
with torch.no_grad() :
for i,(data, target)in enumerate(valid_loader):
data = Variable(data ).cuda()
target = Variable(target ).cuda()
output = cnn_model(data)
 loss += F.cross_entropy(output, target, reduction='sum' ).item()
_, preds = torch.max(output, 1)
 running_corrects += torch.sum(preds == target.data)
 loss /= len(valid_loader.dataset)
 test_losses.append(loss)
 epoch_acc = 100.* running_corrects.double() / len(valid_loader.dataset)
 print('
Average Val Loss: {:.4f}, Val Accuracy: {}/{}({:.3f}%)
 '.format(
 loss, running_corrects, len(valid_loader.dataset), epoch_acc))
if epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(cnn_model.state_dict())
 exp_lr_scheduler.step(loss)
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
def stat_pool(x):
mean, var = tf.nn.moments(x, axes=[1, 2])
 return K.concatenate([mean, var], axis=1)
 def RFCX_MODEL() :
waveform = Input(shape=(None,None,3), dtype=tf.float32)
 noisy_waveform = GaussianNoise(0.2 )(waveform)
 model = efn.EfficientNetB4(include_top=False, weights='imagenet',)
 model_output = model(noisy_waveform)
 model_output = GlobalAveragePooling2D()(model_output)
 dense = Dropout(params.dropout )(model_output)
 predictions = Dense(params.num_classes, activation = params.classifier_activation )(dense)
 model = Model(
 name='Efficientnet', inputs=waveform,
outputs=[predictions])
 return model<compute_test_metric>
cnn_model.eval()
test_preds = None
test_preds = torch.LongTensor()
for i, data in enumerate(test_loader):
data = Variable(data ).cuda()
output = cnn_model(data)
 preds = output.cpu().data.max(1, keepdim=True)[1]
test_preds = torch.cat(( test_preds, preds), dim=0)
<compute_test_metric><EOS> 
out_df = pd.DataFrame({'ImageId':np.arange(1, len(test_loader.dataset)+1), 'Label':test_preds.numpy().squeeze() })
 out_df.to_csv('submission.csv', index=False )