Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<create_dataframe>
from keras import models
from keras import layers
from keras.layers import Dense, Conv2D, Flatten,MaxPooling2D,Dropout,ZeroPadding2D,BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
import pandas as pd
import numpy as np
from matplotlib import pyplot
import os
import zipfile
import tensorflow as tf
from tensorflow.keras.optimizers import RMSprop,Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import random
def get_dataset(filenames, training = True):
if training:
dataset = load_dataset(filenames , training = True)
 dataset = dataset.shuffle(256 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 else:
dataset = load_dataset(filenames , training = False)
 dataset = dataset.batch(GLOBAL_BATCH_SIZE ).cache()
dataset = dataset.prefetch(AUTO)
 return dataset<groupby>
Train = pd.read_csv(".. /input/train.csv")
 Test = pd.read_csv(".. /input/test.csv" )
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
y_true = tf.reshape(y_true, tf.shape(y_pred))
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )<normalization>
y_train = Train['label'].values
train_images = Train.loc[:,Train.columns != 'label'].values
train_images = train_images.reshape(-1,28,28,1)
 train_images.shape
def cosine_decay_with_warmup(global_step,
learning_rate_base,
total_steps,
warmup_learning_rate=0.0,
warmup_steps= 0,
hold_base_rate_steps=0):
if total_steps < warmup_steps:
raise ValueError('total_steps must be larger or equal to '
'warmup_steps.')
 learning_rate = 0.5 * learning_rate_base *(1 + tf.cos(
 np.pi * 
(tf.cast(global_step, tf.float32)- warmup_steps - hold_base_rate_steps 
)/ float(total_steps - warmup_steps - hold_base_rate_steps)))
 if hold_base_rate_steps > 0:
learning_rate = tf.where(
 global_step > warmup_steps + hold_base_rate_steps,
learning_rate, learning_rate_base)
 if warmup_steps > 0:
if learning_rate_base < warmup_learning_rate:
raise ValueError('learning_rate_base must be larger or equal to '
'warmup_learning_rate.')
 slope =(learning_rate_base - warmup_learning_rate)/ warmup_steps
warmup_rate = slope * tf.cast(global_step,
tf.float32)+ warmup_learning_rate
learning_rate = tf.where(global_step < warmup_steps, warmup_rate,
learning_rate)
 return tf.where(global_step > total_steps, 0.0, learning_rate,
name='learning_rate')
 rng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH)) ]
WARMUP_STEPS = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 y = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS)for x in rng]
sns.set(style='whitegrid')
 fig, ax = plt.subplots(figsize=(20, 6))
plt.plot(rng, y )<choose_model_class>
train_images = train_images.astype('float32')/255
test_images_final = test_images_final.astype('float32')/255
class WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):
def __init__(self,
learning_rate_base,
total_steps,
global_step_init=0,
warmup_learning_rate=0.0,
warmup_steps=0,
hold_base_rate_steps=0,
verbose=0):
super(WarmUpCosineDecayScheduler, self ).__init__()
self.learning_rate_base = learning_rate_base
self.total_steps = total_steps
self.global_step = global_step_init
self.warmup_learning_rate = warmup_learning_rate
self.warmup_steps = warmup_steps
self.hold_base_rate_steps = hold_base_rate_steps
self.verbose = verbose
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.global_step = self.global_step + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
lr = cosine_decay_with_warmup(global_step=self.global_step,
learning_rate_base=self.learning_rate_base,
total_steps=self.total_steps,
warmup_learning_rate=self.warmup_learning_rate,
warmup_steps=self.warmup_steps,
hold_base_rate_steps=self.hold_base_rate_steps)
 K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: setting learning '
'rate to %s.' %(self.global_step + 1, lr.numpy()))
total_steps = int(EPOCHS * STEPS_PER_EPOCH)
 warmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 warmup_learning_rate = WARMUP_LEARNING_RATE<choose_model_class>
network = tf.keras.models.Sequential([
tf.keras.layers.ZeroPadding2D(padding=(1, 1), data_format=None,input_shape=(28, 28, 1)) ,
tf.keras.layers.Conv2D(32,(5,5), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ZeroPadding2D(padding=(1, 1), data_format=None),
tf.keras.layers.Conv2D(64,(6,6), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ZeroPadding2D(padding=(1, 1), data_format=None),
tf.keras.layers.Conv2D(64,(7,7), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D(5,5),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.45),
tf.keras.layers.Dense(10, activation='softmax')
 ] )
def RFCX_MODEL() :
waveform = Input(shape=(None,None,3), dtype=tf.float32)
 noisy_waveform = GaussianNoise(0.2 )(waveform)
 model = efn.EfficientNetB2(include_top=False, weights='imagenet',)
 model_output = model(noisy_waveform)
 model_output = GlobalAveragePooling2D()(model_output)
 dense = Dropout(params.dropout )(model_output)
 predictions = Dense(params.num_classes, activation = params.classifier_activation )(dense)
 model = Model(
 name='Efficientnet', inputs=waveform,
outputs=[predictions])
 return model<choose_model_class>
network.compile(optimizer=Adam(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'] )
def get_model() :
with strategy.scope() :
model = RFCX_MODEL()
model.summary()
model.compile(optimizer = 'adam',
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.1),
metrics = [LWLRAP(num_classes = params.num_classes),
])
 return model<define_variables>
X_train, X_test, y_train, y_test = train_test_split(train_images, y_train, test_size=0.15 )
skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)
 oof_pred = []; oof_labels = []; history_list = []
for fold,(idxT, idxV)in enumerate(skf.split(np.arange(10))):
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print(f'
FOLD: {fold+1}')
 print(f'TRAIN: {idxT} VALID: {idxV}')
 TRAIN_FILENAMES = [FILENAMES[x] for x in idxT]
VALID_FILENAMES = [FILENAMES[x] for x in idxV]
np.random.shuffle(TRAIN_FILENAMES)
 train_dataset = get_dataset(TRAIN_FILENAMES, training=True,)
 validation_data= get_dataset(VALID_FILENAMES, training=False)
 model = get_model()
model_path = f'RFCX_model_fold {fold}.h5'
early_stopping = EarlyStopping(monitor = 'val_lwlrap', mode = 'max',
patience = PATIENCE, restore_best_weights=True, verbose=1)
 cosine_warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base= LEARNING_RATE,
total_steps= total_steps,
warmup_learning_rate= warmup_learning_rate,
warmup_steps= warmup_steps,
hold_base_rate_steps=0)
 history = model.fit(train_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[early_stopping, cosine_warm_up_lr],
epochs=EPOCHS,
validation_data = validation_data,
verbose = 2 ).history
history_list.append(history)
 model.save_weights(model_path)
 ds_valid = get_dataset(VALID_FILENAMES, training = False)
 oof_labels.append([target.numpy() for frame, target in iter(ds_valid.unbatch())])
 x_oof = ds_valid.map(lambda frames, target: frames)
 oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))
print(f"<create_dataframe>
train_labels = to_categorical(y_train)
 test_labels = to_categorical(y_test)
 y_train = train_labels
y_train.shape
def get_test_dataset(filenames, training = False):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_unlabeled_tfrecord , num_parallel_calls = AUTO ).unbatch()
dataset = dataset.map(lambda spec : waveform_to_log_mel_spectrogram(spec['audio_wav'], spec['recording_id']), num_parallel_calls = AUTO)
 dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 return dataset.batch(GLOBAL_BATCH_SIZE*4 ).cache()<predict_on_test>
datagen = ImageDataGenerator(rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1)
 datagen.fit(X_train)
 np.concatenate(( X_train,X_train),axis=0)
 random.seed(12345)
 for X_batch, y_batch in datagen.flow(np.concatenate(( X_train,X_train),axis=0), np.concatenate(( y_train,y_train),axis=0), batch_size=35700):
break
X_trainE = X_batch
y_trainE = y_batch
for i in range(0, 9):
pyplot.subplot(330 + 1 + i)
 pyplot.imshow(X_trainE[i].reshape(28, 28), cmap=pyplot.get_cmap('gray'))
pyplot.show()
test_predict = []
test_data = get_test_dataset(TEST_FILES, training = False)
 test_audio = test_data.map(lambda frames, recording_id: frames)
 for fold in range(N_FOLDS):
model.load_weights(f'./RFCX_model_fold {fold}.h5')
 test_predict.append(model.predict(test_audio, verbose = 1))<load_from_csv>
X_Combine = np.concatenate(( X_train,X_trainE),axis=0)
 y_combine = np.concatenate(( y_train,y_trainE),axis=0)
 print(X_Combine.shape)
 print(y_combine.shape )
SUB = pd.read_csv('.. /input/rfcx-species-audio-detection/sample_submission.csv')
 predict = np.array(test_predict ).reshape(N_FOLDS, len(SUB), 60 // TIME, params.num_classes)
 predict = np.mean(np.max(predict ,axis = 2), axis = 0)
 recording_id = test_data.map(lambda frames, recording_id: recording_id ).unbatch()
test_ids = next(iter(recording_id.batch(len(SUB)* 60 // TIME)) ).numpy().astype('U' ).reshape(len(SUB), 60 // TIME)
 pred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],
**{f's{i}' : predict[:, i] for i in range(params.num_classes)} } )<save_to_csv>
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('val_acc')>=0.9930):
print("
Reached 99.0% accuracy so cancelling training!")
 self.model.stop_training = True
callbacks = myCallback()
pred_df.sort_values('recording_id', inplace = True)
 pred_df.to_csv('submission.csv', index = False )<install_modules>
history = network.fit(X_Combine, y_combine, epochs=60, batch_size=512,validation_data=(X_test,test_labels),callbacks=[callbacks] )
!pip install keras<set_options>
test_loss, test_acc = network.evaluate(X_test, test_labels)
 print('test_acc:', test_acc )
warnings.filterwarnings("ignore")
<load_from_csv>
predict = network.predict(test_images_final)
 y_classes = [np.argmax(y, axis=None, out=None)for y in predict]
<categorify><EOS> 
x = list(range(1, 28001))
df = pd.DataFrame({'ImageId' :x,'Label':y_classes})
 df.to_csv("output.csv",index=False)