Kaggle competition: Kannada MNIST
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon())) 
<choose_model_class>
train=pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test=pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 submission_sample = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 train_CNN=train
Gtrain=train
label0=Gtrain.groupby('label' ).get_group(0)
 label1=Gtrain.groupby('label' ).get_group(1)
 label2=Gtrain.groupby('label' ).get_group(2)
 label3=Gtrain.groupby('label' ).get_group(3)
 label4=Gtrain.groupby('label' ).get_group(4)
 label5=Gtrain.groupby('label' ).get_group(5)
 label6=Gtrain.groupby('label' ).get_group(6)
 label7=Gtrain.groupby('label' ).get_group(7)
 label8=Gtrain.groupby('label' ).get_group(8)
 label9=Gtrain.groupby('label' ).get_group(9)
 X_label0=label0.drop('label',axis=1)
 X_label1=label1.drop('label',axis=1)
 X_label2=label2.drop('label',axis=1)
 X_label3=label3.drop('label',axis=1)
 X_label4=label4.drop('label',axis=1)
 X_label5=label5.drop('label',axis=1)
 X_label6=label6.drop('label',axis=1)
 X_label7=label7.drop('label',axis=1)
 X_label8=label8.drop('label',axis=1)
 X_label9=label9.drop('label',axis=1)
 Y_label0=label0.label
Y_label1=label1.label
Y_label2=label2.label
Y_label3=label3.label
Y_label4=label4.label
Y_label5=label5.label
Y_label6=label6.label
Y_label7=label7.label
Y_label8=label8.label
Y_label9=label9.label
X_label0=np.asarray(X_label0,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label1=np.asarray(X_label1,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label2=np.asarray(X_label2,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label3=np.asarray(X_label3,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label4=np.asarray(X_label4,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label5=np.asarray(X_label5,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label6=np.asarray(X_label6,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label7=np.asarray(X_label7,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label8=np.asarray(X_label8,dtype=np.float32 ).reshape(-1,28,28,1)
 X_label9=np.asarray(X_label9,dtype=np.float32 ).reshape(-1,28,28,1)
 test=test.drop('id',axis=1)
 y=train.label.value_counts()
X_train=train.drop('label',axis=1)
 Y_train=train.label
X_train=np.array(X_train,dtype=np.float32)
 Y_train=np.array(Y_train)
 X_train=X_train/255
test=test/255
X_train=X_train.reshape(-1,28,28,1)
 new_image_set=X_train
test=test.values.reshape(-1,28,28,1)
 X_train,X_test,y_train,y_test=train_test_split(X_train,Y_train,random_state=42,test_size=0.15)
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='glorot_uniform', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',f1] )<train_model>
train_CNN_X=train_CNN.drop('label',axis=1)
 train_CNN_Y=train.label
train_CNN_X=np.array(train_CNN_X,dtype=np.float32)
 train_CNN_Y=np.array(train_CNN_Y)
 train_CNN_X=train_CNN_X/255
model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))<predict_on_test>
BUFFER_SIZE = 6000
BATCH_SIZE = 256
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))<predict_on_test>
def make_generator_model() :
model = tf.keras.Sequential()
model.add(layers.Dense(7*7*256, use_bias=False, input_shape=(100,)))
 model.add(layers.BatchNormalization())
 model.add(layers.LeakyReLU())
 model.add(layers.Reshape(( 7, 7, 256)))
 assert model.output_shape ==(None, 7, 7, 256)
 model.add(layers.Conv2DTranspose(128,(5, 5), strides=(1, 1), padding='same', use_bias=False))
assert model.output_shape ==(None, 7, 7, 128)
 model.add(layers.BatchNormalization())
 model.add(layers.LeakyReLU())
 model.add(layers.Conv2DTranspose(64,(5, 5), strides=(2, 2), padding='same', use_bias=False))
assert model.output_shape ==(None, 14, 14, 64)
 model.add(layers.BatchNormalization())
 model.add(layers.LeakyReLU())
 model.add(layers.Conv2DTranspose(1,(5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))
assert model.output_shape ==(None, 28, 28, 1)
 return model
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
def make_discriminator_model() :
model = tf.keras.Sequential()
model.add(layers.Conv2D(64,(5, 5), strides=(2, 2), padding='same',
input_shape=[28, 28, 1]))
model.add(layers.LeakyReLU())
 model.add(layers.Dropout(0.3))
model.add(layers.Conv2D(128,(5, 5), strides=(2, 2), padding='same'))
model.add(layers.LeakyReLU())
 model.add(layers.Dropout(0.3))
model.add(layers.Flatten())
 model.add(layers.Dense(1))
return model
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<set_options>
discriminator = make_discriminator_model()
decision = discriminator(generated_image)
 print(decision )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
def discriminator_loss(real_output, fake_output):
real_loss = cross_entropy(tf.ones_like(real_output), real_output)
 fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)
 total_loss = real_loss + fake_loss
return total_loss
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
def generator_loss(fake_output):
return cross_entropy(tf.ones_like(fake_output), fake_output )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
generator_optimizer = tf.keras.optimizers.Adam(1e-4)
 discriminator_optimizer = tf.keras.optimizers.Adam(1e-4 )
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
checkpoint_dir = './training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
 checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
discriminator_optimizer=discriminator_optimizer,
generator=generator,
discriminator=discriminator )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
EPOCHS = 75
noise_dim = 100
num_examples_to_generate = 6000
seed = tf.random.normal([num_examples_to_generate, noise_dim] )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
@tf.function
def train_step(images):
noise = tf.random.normal([BATCH_SIZE, noise_dim])
 with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
generated_images = generator(noise, training=True)
 real_output = discriminator(images, training=True)
 fake_output = discriminator(generated_images, training=True)
 gen_loss = generator_loss(fake_output)
 disc_loss = discriminator_loss(real_output, fake_output)
 gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)
 gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)
 generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))
discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
def train(dataset, epochs):
for epoch in range(epochs):
start = time.time()
for image_batch in dataset:
train_step(image_batch)
 display.clear_output(wait=True)
 generate_and_save_images(generator,
epoch + 1,
seed)
 if(epoch + 1)% 15 == 0:
checkpoint.save(file_prefix = checkpoint_prefix)
 print('Time for epoch {} is {} sec'.format(epoch + 1, time.time() -start))
display.clear_output(wait=True)
 new_images=generate_and_save_images(generator,
epochs,
seed)
 return new_images
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
%%time
train_dataset = tf.data.Dataset.from_tensor_slices(X_label0 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images0=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label1 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images1=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label2 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images2=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label3 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images3=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label4 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images4=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label5 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images5=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label6 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images6=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label7 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images7=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label8 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images8=train(train_dataset, EPOCHS)
 train_dataset = tf.data.Dataset.from_tensor_slices(X_label9 ).shuffle(BUFFER_SIZE ).batch(BATCH_SIZE)
 gen_images9=train(train_dataset, EPOCHS)
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
gen_images=np.vstack(( gen_images0,gen_images1,gen_images2,gen_images3,gen_images4,gen_images5,gen_images6,gen_images7,gen_images8,gen_images9))
print(gen_images.shape)
 Y_labels=np.vstack(( Y_label0,Y_label1,Y_label2,Y_label3,Y_label4,Y_label5,Y_label6,Y_label7,Y_label8,Y_label9,))
Y_labels=Y_labels.reshape(60000)
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
train_CNN_X=np.vstack(( train_CNN_X.reshape(-1,28,28,1),gen_images.reshape(-1,28,28,1)))
 print(train_CNN_X.shape)
 train_CNN_Y=np.vstack(( train_CNN_Y,Y_labels))
train_CNN_Y=train_CNN_Y.reshape(120000)
 print(train_CNN_Y.shape )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
model = keras.Sequential([
keras.layers.Conv2D(filters=32, kernel_size=(5, 5),padding='Same',activation='relu',input_shape=(28,28,1)) ,
keras.layers.Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',),
keras.layers.BatchNormalization(momentum=.15),
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.Dropout(0.25),
keras.layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'),
keras.layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'),
keras.layers.BatchNormalization(momentum=0.15),
keras.layers.MaxPool2D(pool_size=(2,2), strides=(2,2)) ,
keras.layers.Dropout(0.25),
keras.layers.Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)) ,
keras.layers.Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu'),
keras.layers.BatchNormalization(momentum=.15),
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.Dropout(0.25),
keras.layers.Flatten() ,
keras.layers.Dense(256, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(10, activation='softmax')
 ])
 model.summary()
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
optimizer=keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)
 model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] )
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
model.fit(train_CNN_X, train_CNN_Y, epochs=75)
 Test accuracy:', test_acc )
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
test1=test.reshape(-1,28,28,1)
 prediction1=model.predict(test1)
 prediction1=np.argmax(prediction1,axis=1)
 submission_sample['label']=prediction1
submission_sample.to_csv('submission.csv',index=False)