Kaggle competition: Quora Insincere Questions Classification
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
with strategy.scope() :
model1 = get_model(DenseNet201)
 model1.load_weights("/kaggle/input/more-data-with-densenet220011/my_ef_net_b7.h5" )<choose_model_class>
embed_size = 300
max_features = 120000
maxlen = 70
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
with strategy.scope() :
model2 = get_model(EfficientNetB1)
 model2.load_weights("/kaggle/input/fork-of-more-data-with-effnetb1/my_ef_net_b7.h5" )<predict_on_test>
import os
import re
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Conv1D, MaxPool1D
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.layers import Lambda, Dot
from keras.layers import MaxPool1D, AveragePooling1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
val_dataset = get_validation_dataset()
images_ds = val_dataset.map(lambda image, label: image)
 labels_ds = val_dataset.map(lambda image, label: label ).unbatch()
val_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m1 = model1.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
val_probabilities = alpha*m1+(1-alpha)*m2
val_predictions = np.argmax(val_probabilities, axis=-1)
 scores.append(f1_score(val_labels, val_predictions, labels=range(104), average='macro'))
best_alpha = np.argmax(scores)/100
print('Best alpha: ' + str(best_alpha))<predict_on_test>
assert len(K.tensorflow_backend._get_available_gpus())> 0
K.tensorflow_backend._get_available_gpus()
os.environ['PYTHONHASHSEED'] = '0'
seed(1)
 set_random_seed(2)
 rn.seed(7 )
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities1 = model1.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 probabilities = best_alpha * probabilities1 +(1 - best_alpha)* probabilities2
predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<import_modules>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def split_text(x):
x = wordninja.split(x)
 return '-'.join(x)
 def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
print("Tensorflow version " + tf.__version__ )<find_best_params>
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index, test_df['qid']
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')
 print(GCS_DS_PATH )<define_variables>
def focal_loss(y_true, y_pred):
gamma = 0.1
true_loss = K.mean(y_true *(( 1-y_pred)**gamma * K.log(y_pred + K.epsilon())) , axis=0)
 false_loss = K.mean(( 1-y_true)*(y_pred**gamma * K.log(1-y_pred + K.epsilon())) , axis=0)
 return -(true_loss + false_loss )
IMAGE_SIZE = [512, 512]
GCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-512x512'
AUTO = tf.data.experimental.AUTOTUNE
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset<train_model>
def model_cnn_1d(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv1D(num_filters, kernel_size=filter_sizes[i],
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool1D(pool_size=maxlen - filter_sizes[i] + 1 )(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss=focal_loss, optimizer='adam', metrics=['accuracy'])
 return model
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
ds_train = get_training_dataset()
ds_valid = get_validation_dataset()
ds_test = get_test_dataset()
print("Training:", ds_train)
 print("Validation:", ds_valid)
 print("Test:", ds_test )<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
np.set_printoptions(threshold=10, linewidth=80)
 print("Training data shapes:")
 for image, label in ds_train.take(3):
print(image.numpy().shape, label.numpy().shape)
 print("Training data label examples:", label.numpy() )<data_type_conversions>
def model_lstm_atten_mix(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="elu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
def model_lstm_atten_glove(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Dropout(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
print("Test data shapes:")
 for image, idnum in ds_test.take(3):
print(image.numpy().shape, idnum.numpy().shape)
 print("Test data IDs:", idnum.numpy().astype('U'))<categorify>
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )<define_variables>
def model_my_lstm(embedding_matrix):
p_drop = 0.1
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size*2, weights=[embedding_matrix], trainable=False )(inp)
 x = Dropout(p_drop )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Dropout(p_drop )(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 atten_x = Attention(maxlen )(x)
 max_x = MaxPool1D(maxlen )(x)
 max_x = Flatten()(max_x)
 ave_x = AveragePooling1D(maxlen )(x)
 ave_x = Flatten()(ave_x)
 x = Concatenate()([atten_x, max_x, ave_x])
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss=focal_loss, optimizer='adam', metrics=['accuracy'])
 return model
ds_iter = iter(ds_train.unbatch().batch(20))<define_variables>
def train_pred(model, epochs):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, verbose=2)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_test_y
def train_pred_snapshot(model, epochs, start_epoch):
pred_list = []
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, verbose=2)
 if e >= start_epoch:
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 pred_list.append(pred_test_y)
 return pred_list
one_batch = next(ds_iter)
 display_batch_of_images(one_batch )<install_modules>
%%time
train_X, test_X, train_y, word_index, test_df = load_and_prec()
!pip install -q efficientnet 
<choose_model_class>
%%time
embedding_matrix_glove = load_glove(word_index )
with strategy.scope() :
enet = efn.EfficientNetB7(input_shape=[*IMAGE_SIZE, 3], weights='noisy-student', include_top=False)
 enet.trainable = True
model1 = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
<define_variables>
%%time
embedding_matrix_para = load_para(word_index )
EPOCHS=100<choose_model_class>
embedding_matrix_mix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis = 0)
 embedding_matrix_concat = np.concatenate([embedding_matrix_glove, embedding_matrix_para], axis = 1 )
model1.compile(
 optimizer=tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model1.summary()<choose_model_class>
%%time
outputs = []
pred_test_y = train_pred(model_cnn_1d(embedding_matrix_glove), epochs = 2)
 outputs.append([pred_test_y, '1d-CNN'] )
def exponential_lr(epoch,
start_lr = 0.00001, min_lr = 0.00001, max_lr = 0.00005,
rampup_epochs = 5, sustain_epochs = 0,
exp_decay = 0.8):
def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):
if epoch < rampup_epochs:
lr =(( max_lr - start_lr)/
rampup_epochs * epoch + start_lr)
 elif epoch < rampup_epochs + sustain_epochs:
lr = max_lr
else:
lr =(( max_lr - min_lr)*
exp_decay**(epoch - rampup_epochs - sustain_epochs)+
min_lr)
 return lr
return lr(epoch,
start_lr,
min_lr,
max_lr,
rampup_epochs,
sustain_epochs,
exp_decay)
 lr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [exponential_lr(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<train_model>
%%time
pred_test_y = train_pred(model_lstm_du(embedding_matrix_mix), epochs = 2)
 outputs.append([pred_test_y, 'LSTM-DU'] )
EPOCHS = 21
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES//256
history = model1.fit(
 ds_train,
validation_data=ds_valid,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[lr_callback],
)<predict_on_test>
%%time
pred_test_y = train_pred(model_lstm_atten_mix(embedding_matrix_mix), epochs = 3)
 outputs.append([pred_test_y, '2-LSTM-attention-mix'] )
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities = model1.predict(images_ds)
 cm_predictions = np.argmax(cm_probabilities, axis=-1)
 labels = range(len(CLASSES))
cmat = confusion_matrix(
 cm_correct_labels,
cm_predictions,
labels=labels, 
)
 cmat =(cmat.T / cmat.sum(axis=1)).T<compute_train_metric>
%%time
pred_test_y = train_pred(model_lstm_atten_glove(embedding_matrix_glove), epochs = 3)
 outputs.append([pred_test_y, '2-LSTM-attention-glov'] )
score = f1_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 precision = precision_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 recall = recall_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 display_confusion_matrix(cmat, score, precision, recall )<define_variables>
%%time
pred_test_y_list = train_pred_snapshot(model_my_lstm(embedding_matrix_concat), epochs=5, start_epoch=3)
 outputs.extend([[pred, 'my_lstm'] for pred in pred_test_y_list] )
dataset = get_validation_dataset()
dataset = dataset.unbatch().batch(20)
 batch = iter(dataset )<predict_on_test>
%%time
w_list = [0.15573309, 0.15155236, 0.20362984, 0.1206986, 0.14503201, 0.2233541]
w_normed = np.array(w_list)/ np.sum(w_list)
 pred_test_y = np.sum([outputs[i][0]*w_normed[i] for i in range(len(w_normed)) ], axis = 0)
 pred_test_y =(pred_test_y > 0.3434 ).astype(int )
<predict_on_test><EOS> 
%%time
out_df = pd.DataFrame({"qid":test_df.values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )