Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<save_to_csv>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras import backend as K
from tensorflow.keras import initializers, layers
from sklearn.model_selection import train_test_split
test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = np.array(ans)
 out_df.to_csv("submission.csv", index=False )<load_pretrained>
seed = 18
np.random.seed(seed )
def save_model(model, model_path):

 torch.save(model.state_dict() , model_path)
 def load_model(model, model_path, use_cuda=False):

 map_location = 'cpu'
if use_cuda and torch.cuda.is_available() :
map_location = 'cuda:0'
model.load_state_dict(torch.load(model_path, map_location))
return model<categorify>
warnings.filterwarnings('ignore' )
NEG_INF = -10000
TINY_FLOAT = 1e-6
def mask_softmax(matrix, mask=None):

 if mask is None:
result = F.softmax(matrix, dim=-1)
 else:
mask_norm =(( 1 - mask)* NEG_INF ).to(matrix)
 for i in range(matrix.dim() - mask_norm.dim()):
mask_norm = mask_norm.unsqueeze(1)
 result = F.softmax(matrix + mask_norm, dim=-1)
 return result
def mask_mean(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 mask_sum = torch.sum(
 seq * mask.unsqueeze(-1 ).float() , dim=1)
 seq_len = torch.sum(mask, dim=-1)
 mask_mean = mask_sum /(seq_len.unsqueeze(-1 ).float() + TINY_FLOAT)
 return mask_mean
def mask_max(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 torch
mask_max, _ = torch.max(
 seq +(1 - mask.unsqueeze(-1 ).float())* NEG_INF,
dim=1)
 return mask_max
def seq_mask(seq_len, max_len):

 idx = torch.arange(max_len ).to(seq_len ).repeat(seq_len.size(0), 1)
 mask = torch.gt(seq_len.unsqueeze(1), idx ).to(seq_len)
 return mask
class DynamicLSTM(nn.Module):

 def __init__(self, input_size, hidden_size=100,
num_layers=1, dropout=0., bidirectional=False):
super(DynamicLSTM, self ).__init__()
self.lstm = nn.LSTM(
 input_size, hidden_size, num_layers, bias=True,
batch_first=True, dropout=dropout, bidirectional=bidirectional)
 def forward(self, x, seq_lens):
_, idx_sort = torch.sort(seq_lens, dim=0, descending=True)
 _, idx_unsort = torch.sort(idx_sort, dim=0)
 x_sort = torch.index_select(x, dim=0, index=idx_sort)
 seq_lens_sort = torch.index_select(seq_lens, dim=0, index=idx_sort)
 x_packed = pack_padded_sequence(
 x_sort, seq_lens_sort, batch_first=True)
 y_packed, _ = self.lstm(x_packed)
 y_sort, length = pad_packed_sequence(y_packed, batch_first=True)
 y = torch.index_select(y_sort, dim=0, index=idx_unsort)
 return y
class QuoraModel(nn.Module):

 def __init__(self, args):
super(QuoraModel, self ).__init__()
vocab_size = args["vocab_size"]
pretrained_embed = args["pretrained_embed"]
padding_idx = args["padding_idx"]
embed_dim = 300
num_classes = 1
num_layers = 2
hidden_dim = 50
dropout = 0.5
if pretrained_embed is None:
self.embed = nn.Embedding(vocab_size, embed_dim)
 else:
self.embed = nn.Embedding.from_pretrained(
 pretrained_embed, freeze=False)
 self.embed.padding_idx = padding_idx
self.rnn = DynamicLSTM(
 embed_dim, hidden_dim, num_layers=num_layers,
dropout=dropout, bidirectional=True)
 self.fc_att = nn.Linear(hidden_dim * 2, 1)
 self.fc = nn.Linear(hidden_dim * 6, hidden_dim)
 self.act = nn.ReLU()
self.drop = nn.Dropout(dropout)
 self.out = nn.Linear(hidden_dim, num_classes)
 self.loss = nn.BCEWithLogitsLoss()
def forward(self, word_seq, seq_len):
max_seq_len = torch.max(seq_len)
 mask = seq_mask(seq_len, max_seq_len)
 e = self.drop(self.embed(word_seq))
r = self.rnn(e, seq_len)
 att = self.fc_att(r ).squeeze(-1)
 att = mask_softmax(att, mask)
 r_att = torch.sum(att.unsqueeze(-1)* r, dim=1)
 r_avg = mask_mean(r, mask)
 r_max = mask_max(r, mask)
 r = torch.cat([r_avg, r_max, r_att], dim=-1)
 f = self.drop(self.act(self.fc(r)))
 logits = self.out(f ).squeeze(-1)
 return logits
class Trainer(object):

 def __init__(self, **kwargs):
self.n_epochs = kwargs["epochs"]
self.batch_size = kwargs["batch_size"]
self.validate = kwargs["validate"]
self.save_best_dev = kwargs["save_best_dev"]
self.use_cuda = kwargs["use_cuda"]
self.print_every_step = kwargs["print_every_step"]
self.optimizer = kwargs["optimizer"]
self.model_path = kwargs["model_path"]
self.eval_metrics = kwargs["eval_metrics"]
self._best_accuracy = 0.0
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def train(self, network, train_data, dev_data=None):
network = network.to(self.device)
 train_iter = torchtext.data.Iterator(
 dataset=train_data, batch_size=self.batch_size,
train=True, shuffle=True, sort=False,
device=self.device)
 if self.validate:
default_valid_args = {
"batch_size": max(8, self.batch_size // 10),
"use_cuda": self.use_cuda}
validator = Tester(**default_valid_args)
 start = time.time()
for epoch in range(1, self.n_epochs + 1):
network.train()
train_iter.init_epoch()
self._train_step(
 train_iter, network, start=start,
n_print=self.print_every_step, epoch=epoch)
 if self.validate:
if dev_data is None:
raise RuntimeError(
 "self.validate is True in trainer, "
"but dev_data is None."
" Please provide the validation data.")
 eval_results = validator.test(network, dev_data)
 if self.save_best_dev and self.best_eval_result(eval_results):
save_model(network, self.model_path)
 print("Saved better model selected by validation.")
 def _train_step(self, data_iterator, network, **kwargs):

 step = 0
for batch in data_iterator: 
(text, text_len), target = batch.text, batch.target
self.optimizer.zero_grad()
logits = network(text, text_len)
 loss = network.loss(logits, target.float())
 loss.backward()
self.optimizer.step()
if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0:
end = time.time()
diff = timedelta(seconds=round(end - kwargs["start"]))
print_output = "[epoch: {:>3} step: {:>4}]" \
" train loss: {:>4.6} time: {}".format(
 kwargs["epoch"], step, loss.item() , diff)
 print(print_output)
 step += 1
def best_eval_result(self, eval_results):

 assert self.eval_metrics in eval_results, \
"Evaluation doesn't contain metrics '{}'." \ 
.format(self.eval_metrics)
 accuracy = eval_results[self.eval_metrics]
if accuracy > self._best_accuracy:
self._best_accuracy = accuracy
return True
else:
return False
class Tester(object):

 def __init__(self, **kwargs):
self.batch_size = kwargs["batch_size"]
self.use_cuda = kwargs["use_cuda"]
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def test(self, network, dev_data, threshold=0.33):
network = network.to(self.device)
 network.eval()
output_list = []
truth_list = []
data_iter = torchtext.data.Iterator(
 dataset=dev_data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text, target = batch.text, batch.target
with torch.no_grad() :
prediction = network(*text)
 output_list.append(prediction.detach())
 truth_list.append(target.detach())
 eval_results = self.evaluate(output_list, truth_list, threshold)
 print("[tester] {}".format(self.print_eval_results(eval_results)))
 return eval_results
def evaluate(self, predict, truth, threshold=0.33):

 y_trues, y_preds = [], []
for y_true, logit in zip(truth, predict):
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_true = y_true.cpu().numpy()
y_trues.append(y_true)
 y_preds.append(y_pred)
 y_true = np.concatenate(y_trues, axis=0)
 y_pred = np.concatenate(y_preds, axis=0)
 precision = metrics.precision_score(y_true, y_pred, pos_label=1)
 recall = metrics.recall_score(y_true, y_pred, pos_label=1)
 f1 = metrics.f1_score(y_true, y_pred, pos_label=1)
 metrics_dict = {"precision": precision, "recall": recall, "f1": f1}
return metrics_dict
def print_eval_results(self, results):

 return ", ".join(
 [str(key)+ "=" + "{:.4f}".format(value)
 for key, value in results.items() ])
 class Predictor(object):

 def __init__(self, batch_size=8, use_cuda=False):
self.batch_size = batch_size
self.use_cuda = use_cuda
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def predict(self, network, data, threshold=0.33):
network = network.to(self.device)
 network.eval()
batch_output = []
data_iter = torchtext.data.Iterator(
 dataset=data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text = batch.text
with torch.no_grad() :
prediction = network(*text)
 batch_output.append(prediction.detach())
 return self._post_processor(batch_output, threshold)
 def _post_processor(self, batch_output, threshold=0.33):

 y_preds = []
for logit in batch_output:
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_preds.append(y_pred)
 y_pred = np.concatenate(y_preds, axis=0)
 return y_pred<define_variables>
dig_mnist = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv")
 sample_submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 test = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 train = pd.read_csv(".. /input/Kannada-MNIST/train.csv" )
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
embed_path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
submission_path = './submission.csv'
model_path = './default_model.pkl'<create_dataframe>
x = train.iloc[:,1:].values
y = train.label.values
def pre() :

 print("Pre-processing...")
 fix_length = 100
text = torchtext.data.Field(
 sequential=True, use_vocab=True, lower=True,
tokenize=nltk.word_tokenize, batch_first=True,
is_target=False, fix_length=fix_length,
include_lengths=True)
 target = torchtext.data.Field(
 sequential=False, use_vocab=False,
batch_first=True, is_target=True)
 dataset = torchtext.data.TabularDataset(
 train_path, format='csv',
fields={"question_text":('text', text),
"target":('target', target)})
 data_test = torchtext.data.TabularDataset(
 test_path, format='csv',
fields={"question_text":('text', text)})
 text.build_vocab(dataset, data_test, min_freq=3)
 text.vocab.load_vectors(torchtext.vocab.Vectors(embed_path))
vocab_size = len(text.vocab.itos)
 padding_idx = text.vocab.stoi[text.pad_token]
data_train, data_val = dataset.split(split_ratio=0.9)
 print("train set size:", len(data_train))
print("val set size:", len(data_val))
print("test set size:", len(data_test))
print("vocab size:", len(text.vocab.itos))
print("embed shape:", text.vocab.vectors.shape)
 print('')
 args_dict = {
"data_train": data_train, "data_val": data_val,
"data_test": data_test, "vocab_size": vocab_size,
"padding_idx": padding_idx}
return args_dict<categorify>
x_test = test.drop("id", axis=1 ).iloc[:,:].values
args = pre()<train_on_grid>
x_dig = dig_mnist.drop('label', axis=1 ).iloc[:,:].values
y_dig = keras.utils.to_categorical(dig_mnist.label )
def train(**args):

 print("Training...")
 data_train = args["data_train"]
pretrained_embed = data_train.fields["text"].vocab.vectors
model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": pretrained_embed,
}
model = QuoraModel(model_args)
 trainer_args = {
"epochs": 4,
"batch_size": 128,
"validate": True,
"save_best_dev": True,
"use_cuda": True,
"print_every_step": 1000,
"optimizer": torch.optim.Adam(model.parameters() , lr=1e-3),
"model_path": model_path,
"eval_metrics": "f1",
}
trainer = Trainer(**trainer_args)
 data_val = args["data_val"]
trainer.train(model, data_train, dev_data=data_val)
 print('' )<train_model>
x = x.reshape(x.shape[0], 28, 28, 1)
 y = keras.utils.to_categorical(y, 10 )
train(**args )<choose_model_class>
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.1, random_state = seed )
def test(**args):

 print("Testing...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 tester_args = {
"batch_size": 128,
"use_cuda": True,
}
tester = Tester(**tester_args)
 data_val = args["data_val"]
best_thresh, best_f1 = 0., 0.
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 f1 = tester.test(model, data_val, threshold=thresh)["f1"]
print("threshold: {:>.2f} f1: {}".format(thresh, f1))
if f1 > best_f1:
best_thresh, best_f1 = thresh, f1
args["threshold"] = best_thresh
print("best f1 on dev: {} threshold: {:>.2f}".format(best_f1, best_thresh))
print('')
 return args<choose_model_class>
train_datagen = ImageDataGenerator(rescale = 1./255.,
rotation_range = 10,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 10,
zoom_range = 0.1,
horizontal_flip = False )
def infer(**args):

 print("Predicting...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 predictor = Predictor(batch_size=128, use_cuda=False)
 data_test = args["data_test"]
threshold = args["threshold"]
y_pred = predictor.predict(model, data_test, threshold=threshold)
 test_df = pd.read_csv(test_path, index_col=False, header=0)
 data = {"qid": test_df["qid"], "prediction": y_pred}
subm_df = pd.DataFrame(data=data)
 subm_df.to_csv(submission_path, header=True, index=False)
 print("submission saved as {}.".format(submission_path))
print('' )<import_modules>
val_datagen = ImageDataGenerator(rescale=1./255.)
tqdm.pandas() 
<load_from_csv>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding="same", input_shape=(28,28,1)) ,
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding="same"),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256,(3,3), padding="same"),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding="same"),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation="softmax")
 ] )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )<feature_engineering>
initial_learningrate=0.001
batch_size = 1024
epochs = 50
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()<define_variables>
optimizer = RMSprop(learning_rate=initial_learningrate,
momentum=0.1,
centered=True,
name="RMSprop" )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<feature_engineering>
model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=['accuracy'] )
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))<split>
learning_rate_reduction = ReduceLROnPlateau(monitor="loss",
patience=5,
verbose=1,
factor=0.2,
min_lr=0.00001)
 es = EarlyStopping(monitor="val_loss", mode="min", verbose=1, patience=20, restore_best_weights=True )
train_df, val_df = train_test_split(train, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = None
maxlen = 72
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=None, filters='')
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
sub = test[['qid']]<set_options>
history = model.fit_generator(
 train_datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)//batch_size,
epochs=epochs,
validation_data=val_datagen.flow(x_val, y_val),
validation_steps=50,
callbacks=[learning_rate_reduction, es] )
del train, test
gc.collect()<statistical_test>
x_dig = x_dig/255
x_dig = x_dig.reshape(x_dig.shape[0],28,28,1 )
EMBEDDING_FILE_GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE_GLOVE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
np.random.seed(10)
 word_index = tokenizer.word_index
max_features = len(word_index)+1
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
model.evaluate(x_dig,y_dig,verbose=2 )
class AdamW(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- lr * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<choose_model_class>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
%%time
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x1 = Conv1D(filters=96, kernel_size=1, padding='same', activation='relu', kernel_initializer=glorot_normal(1000))(x)
 x2 = Conv1D(filters=72, kernel_size=2, padding='same', activation='relu', kernel_initializer=glorot_normal(2000))(x)
 x3 = Conv1D(filters=48, kernel_size=3, padding='same', activation='relu', kernel_initializer=glorot_normal(3000))(x)
 x4 = Conv1D(filters=24, kernel_size=5, padding='same', activation='relu', kernel_initializer=glorot_normal(4000))(x)
 x1 = GlobalMaxPool1D()(x1)
 x2 = GlobalMaxPool1D()(x2)
 x3 = GlobalMaxPool1D()(x3)
 x4 = GlobalMaxPool1D()(x4)
 merge1 = concatenate([x1, x2, x3, x4])
 x = Dense(200, activation="relu", kernel_initializer=he_uniform(seed=12300))(merge1)
 x = Dropout(0.22 )(x)
 x = BatchNormalization()(x)
 x = Dense(200, activation="relu", kernel_initializer=he_uniform(seed=12300))(x)
 x = Dropout(0.22 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=0.08),)
 print(model.summary() )<train_model>
predictions = model.predict_classes(x_test/255.)
<train_model>
submission['label'] = predictions
%%time
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 callbacks = [checkpoint]
model.fit(train_X, train_y, batch_size=512, epochs=5,
validation_data=(val_X, val_y), verbose=2, callbacks=callbacks,
)<predict_on_test>
submission.to_csv("submission.csv",index=False )