Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_train_metric>
warnings.filterwarnings('ignore' )
class BinaryFocalLoss(nn.Module):
def __init__(self, gamma, alpha=1.) :
super().__init__()
self.gamma = gamma
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss(reduction='none')
 def forward(self, preds, targets):
bce_loss = self.loss_func(preds, targets)
 probas = torch.sigmoid(preds)
 loss = torch.where(
 targets>=0.5,
self.alpha*(( 1.-probas)**self.gamma)*bce_loss, 
(probas**self.gamma)*bce_loss 
)
 loss = loss.mean()
return loss
class MaskedLossWithTPFP(BCEWithLogitsLossFlat):
def __init__(self, **kwargs):
super().__init__(reduction='sum', **kwargs)
 def __call__(self, inp, targ, **kwargs):
weight =(targ != 0.).float().to(targ.device)
 self.func.weight = weight.view(-1)
 targ[targ == -1] = 0.
 n = weight.sum()
masked_sum = super().__call__(inp, targ, **kwargs)
 return masked_sum / n<choose_model_class>
class ReadCsvData(object):
def __init__(self):
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 self._train = df_train.drop(['label'], axis=1 ).values
self._labels = df_train['label'].values
self._test = df_test.values
def get_data(self):
self._train = self._train.astype(np.float32)
 self._train = np.multiply(self._train, 1.0 / 255.0)
 self._test = self._test.astype(np.float32)
 self._test = np.multiply(self._test, 1.0 / 255.0)
 self._labels = np.identity(10)[self._labels]
return self._train, self._labels,self._test
if LOSS_FUNCTION == LossFunction.FOCAL_LOSS.value:
loss_func = BinaryFocalLoss(gamma=2., alpha=1.)
elif LOSS_FUNCTION == LossFunction.BCE_LOGIT_LOSS.value:
loss_func = BCEWithLogitsLossFlat()
elif LOSS_FUNCTION == LossFunction.CE_SOFTMAX_LOSS.value:
loss_func = CrossEntropyLossFlat()
elif LOSS_FUNCTION == LossFunction.MASKED_BCE_WITH_TPFP.value:
loss_func = MaskedLossWithTPFP()
else:
raise ValueError('Invalid loss function')
 if CHANNEL == Channel.SINGLE.value:
config_dict = {'n_in': 1}
elif CHANNEL == Channel.THREE.value:
config_dict = {'n_in': 3}
else:
raise NotImplementedError
cbs = MixUp(1.) if IS_MIXUP else None
if NORMALIZER == Normalizer.IMAGENET_STATS.value:
def encode_tensorimage(self, x:TensorImage):
return(x-self.mean)/ self.std
def encode_audiospec(self, x:AudioSpectrogram):
return(x-self.mean)/ self.std
Normalize.encodes = TypeDispatch([encode_tensorimage, encode_audiospec])
 learner = cnn_learner(
 dls, resnet34,
pretrained=True,
normalize=True,
config=config_dict,
loss_func=loss_func,
metrics=[lwrap_metric],
cbs=cbs 
)
 elif NORMALIZER == Normalizer.SAMPLE_STATS.value:
learner = cnn_learner(
 dls, resnet34,
pretrained=True,
normalize=False,
config=config_dict,
loss_func=loss_func,
metrics=[lwrap_metric],
cbs=cbs 
)
 else:
raise NotImplementedError
print('Learner is set' )<find_best_params>
class DataSet(object):
def __init__(self,
images,
labels):
assert images.shape[0] == labels.shape[0],('images.shape: %s labels.shape: %s' %(images.shape, labels.shape))
self._num_examples = images.shape[0]
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def next_batch(self, batch_size):

 start = self._index_in_epoch
self._index_in_epoch += batch_size
if self._index_in_epoch > self._num_examples:
self._epochs_completed += 1
perm = np.arange(self._num_examples)
 np.random.shuffle(perm)
 self._images = self._images[perm]
self._labels = self._labels[perm]
start = 0
self._index_in_epoch = batch_size
assert batch_size <= self._num_examples
end = self._index_in_epoch
return self._images[start:end], self._labels[start:end]
if(NORMALIZER == Normalizer.IMAGENET_STATS.value):
if CHANNEL ==(Channel.SINGLE.value):
learner.dls.after_batch[-1].mean = learner.dls.after_batch[-1].mean[:, 0:1, :, :]
learner.dls.after_batch[-1].std = learner.dls.after_batch[-1].std[:, 0:1, :, :]
assert learner.dls.after_batch[-1].mean.shape[1] == 1
assert learner.dls.train.after_batch[-1].mean.shape[1] == 1
if CHANNEL ==(Channel.SINGLE.value):
mean=learner.dls.after_batch[-1].mean.cpu().numpy().flatten() [0]
std=learner.dls.after_batch[-1].std.cpu().numpy().flatten() [0]
elif CHANNEL ==(Channel.THREE.value):
mean=learner.dls.after_batch[-1].mean.cpu().numpy().flatten()
std=learner.dls.after_batch[-1].std.cpu().numpy().flatten()
else:
raise NotImplementedError
learner.dls.valid.after_batch.add(
 Normalize.from_stats(mean=mean, std=std),
'after_batch' 
)
 print(f'Train batch_tfms:
{learner.dls.train.after_batch}')
 print(f'Valid batch_tfms:
{learner.dls.valid.after_batch}' )<find_best_params>
sess = tf.InteractiveSession()
x = tf.placeholder(tf.float32, shape=[None, 784])
 y_ = tf.placeholder(tf.float32, shape=[None, 10])
 W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))
W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 50], stddev=0.1))
b_conv1 = tf.Variable(tf.constant(0.1, shape=[50]))
learner.loss_func<train_model>
x_image = tf.reshape(x, [-1,28,28,1])
 h_conv1 = tf.nn.relu(tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME')+ b_conv1)
 h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME' )
learner.save('model_last_epoch')
 print('Model checkpoint saved' )<train_model>
W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 50, 100], stddev=0.1))
b_conv2 = tf.Variable(tf.constant(0.1, shape=[100]))
h_conv2 = tf.nn.relu(tf.nn.conv2d(h_pool1, W_conv2, strides=[1, 1, 1, 1], padding='SAME')+ b_conv2)
 h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME' )
try:
learner.remove_cbs(FetchPredsCallback)
 except:
print('Failed to remove FetchPredsCallback, probably its absent')
 learner.cbs<predict_on_test>
W_fc1 = tf.Variable(tf.truncated_normal([7 * 7 * 100, 1024], stddev=0.1))
b_fc1 = tf.Variable(tf.constant(0.1, shape=[1024]))
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*100])
 h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1)+ b_fc1 )
preds, targs, loss = learner.get_preds(with_loss=True)
 l = loss.mean().cpu().numpy()
print(f'Loss on validation: {l:.2f}' )<init_hyperparams>
rate = tf.placeholder(tf.float32)
 h_fc1_drop = tf.nn.dropout(h_fc1, 1-rate )
def sample_one_subclips(row, i):
length = int(RANDOM_INTERVAL * SR)
 total_length = int(60 * SR)
 if(i + 1)* length > total_length:
_frame_offset = total_length - length
else:
_frame_offset = i * length
audio_fn = test_path.resolve() /f'{row.recording_id}.flac'
audio_subclip = AudioTensor.create(audio_fn,
frame_offset=_frame_offset,
num_frames=length)
 return audio_subclip<load_from_csv>
W_fc2 = tf.Variable(tf.truncated_normal([1024, 10], stddev=0.1))
b_fc2 = tf.Variable(tf.constant(0.1, shape=[10]))
y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2)+ b_fc2 )
test_df = pd.read_csv(DATA_DIR/'sample_submission.csv' )<find_best_params>
cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))
train_step = tf.train.AdamOptimizer(1e-4 ).minimize(cross_entropy)
 correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
predict = tf.argmax(y_conv,1 )
dls.train_ds.tls[0].tfms<train_model>
sess.run(tf.global_variables_initializer() )
dls.valid_ds.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=0))
sample_test_dl = dls.test_dl(test_df, with_labels=False)
 print(sample_test_dl.after_batch )<find_best_model_class>
input_data=ReadCsvData()
x_train, y_label, x_test = input_data.get_data()
subclips_preds = []
segment_n = ceil(( 60*SR)/(RANDOM_INTERVAL*SR))
cp_pipeline = learner.dls.valid_ds.tls[0].tfms
for crt_i in range(segment_n):
print(f'Running prediction on subclip {crt_i} for all test samples...')
 crt_test_dl = learner.dls.test_dl(test_df, with_labels=False)
 crt_test_dl.tls[0].tfms = Pipeline(partial(sample_one_subclips, i=crt_i))
_preds = learner.get_preds(dl=crt_test_dl)
 if IS_MULTILABEL:
_preds = _preds[0]
else:
_preds = torch.nn.Softmax(dim=-1 )(_preds[0])
 subclips_preds.append(_preds )<set_options>
Batch_size=100
Train_Number=5300
accracies = []
crt_test_dl.after_batch<save_to_csv>
print('Start Learning', datetime.now() ,)
 for j in range(3):
train_dataset = DataSet(x_train,y_label)
 for i in range(Train_Number):
batch_x, batch_y = train_dataset.next_batch(Batch_size)
 if i%100 == 0:
train_accuracy = accuracy.eval(feed_dict={x:batch_x, y_: batch_y, rate: 0.0})
 accracies.append(train_accuracy)
 print("step %d, training accuracy %g"%(i, train_accuracy))
train_step.run(feed_dict={x: batch_x, y_: batch_y, rate: 0.5})
 print("step %d, training accuracy %g"%(i, train_accuracy))
print('Finish Learning', datetime.now() , )
<find_best_params><EOS> 
submission_file=pd.DataFrame({'ImageId':np.arange(1,(x_test.shape[0] + 1)) , 'Label':predict.eval(feed_dict={x: x_test, rate: 0.0})})
 print(submission_file)
 submission_file.to_csv("submission_v1.csv", index=False)
 print('Save submission', datetime.now() , )