Kaggle competition: Quora Insincere Questions Classification
es_callback = tf.keras.callbacks.EarlyStopping(min_delta=0, patience=10, verbose=1, mode='auto', restore_best_weights=True )<train_model>
stop_words = set(stopwords.words('english'))
if VALIDATION:
history1 = model1.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=40, validation_data=get_validation_dataset() , callbacks = [lr_callback, es_callback])
 else:
history1 = model1.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=30, callbacks = [lr_callback])
<train_model>
def clean_text(x):
x = str(x)
 for p in string.punctuation + '“…':
x = x.replace(p, ' ' + p + ' ')
 x = x.replace('_', '')
 x = re.sub("`","'", x)
 x = re.sub("(?i)n't",' not', x)
 x = re.sub("(?i)'re",' are', x)
 x = re.sub("(?i)'s",' is', x)
 x = re.sub("(?i)'d",' would', x)
 x = re.sub("(?i)'ll",' will', x)
 x = re.sub("(?i)'t",' not', x)
 x = re.sub("(?i)'ve",' have', x)
 x = re.sub("(?i)'m",' am', x)
 x = re.sub("(?i)n\’t",' not', x)
 x = re.sub("(?i)\’re",' are', x)
 x = re.sub("(?i)\’s",' is', x)
 x = re.sub("(?i)\’d",' would', x)
 x = re.sub("(?i)\’ll",' will', x)
 x = re.sub("(?i)\’t",' not', x)
 x = re.sub("(?i)\’ve",' have', x)
 x = re.sub("(?i)\’m",' am', x)
 x = re.sub('(?i)Quorans','Quora', x)
 x = re.sub('(?i)Qoura','Quora', x)
 x = re.sub('(?i)Quoran','Quora', x)
 x = re.sub('(?i)dropshipping','drop shipping', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 x = re.sub('(?i)WooCommerce','Commerce', x)
 x = re.sub('(?i)hairfall','hair fall', x)
 x = re.sub('(?i)marksheet','mark sheet', x)
 x = re.sub('(?i)articleship','article ship', x)
 x = re.sub('(?i)cryptocurrencies','cryptocurrency', x)
 x = re.sub('(?i)coinbase','cryptocurrency', x)
 x = re.sub('(?i)altcoin','bitcoin', x)
 x = re.sub('(?i)altcoins','bitcoins', x)
 x = re.sub('(?i)litecoin','bitcoin', x)
 x = re.sub('(?i)litecoins','bitcoins', x)
 x = re.sub('(?i)demonetisation','demonetization', x)
 x = re.sub('(?i)ethereum','bitcoin', x)
 x = re.sub('(?i)ethereums','bitcoins', x)
 x = re.sub('(?i)quorans','quora', x)
 x = re.sub('(?i)Brexit','britan exit', x)
 x = re.sub('(?i)upwork','freelance', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)Blockchain','blockchain', x)
 x = re.sub('(?i)GDPR','General Data Protection Regulation', x)
 x = re.sub('(?i)Qoura','quora', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Cryptocurrency','cryptocurrency', x)
 x = re.sub('(?i)Binance','cryptocurrency', x)
 x = re.sub('(?i)Redmi','mobile phone', x)
 x = re.sub('(?i)TensorFlow','Tensor Flow', x)
 x = re.sub('(?i)Golang','programming language', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 return x
if VALIDATION:
history2 = model2.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=40, validation_data=get_validation_dataset() , callbacks = [lr_callback, es_callback])
 else:
history2 = model2.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=30, callbacks = [lr_callback] )<predict_on_test>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 print('Train shape : ', train_df.shape)
 print('Test shape : ', test_df.shape )
if VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities =(model1.predict(images)+model2.predict(images)) /2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 print("Correct labels: ", cm_correct_labels.shape, cm_correct_labels)
 print("Predicted labels: ", cm_predictions.shape, cm_predictions )<compute_test_metric>
train_df, val_df = train_test_split(train_df, test_size=0.2,random_state=123 )
if VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<predict_on_test>
tqdm_notebook.pandas(desc="Example Desc")
 train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities =(model1.predict(test_images_ds)*0.3)+model2.predict(test_images_ds)*0.7
predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 !head submission.csv<define_variables>
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
if VALIDATION:
dataset = get_validation_dataset()
dataset = dataset.unbatch().batch(20)
 batch = iter(dataset )<predict_on_test>
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))
if VALIDATION:
images, labels = next(batch)
 probabilities =(model1.predict(images)*0.3)+model2.predict(images)*0.7
predictions = np.argmax(probabilities, axis=-1)
 display_batch_of_images(( images, labels), predictions )<install_modules>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
!pip install -q efficientnet<import_modules>
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))
print("Tensorflow version " + tf.__version__)
<set_options>
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
IMAGE_SIZE = [512, 512]
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<init_hyperparams>
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: replace_typical_misspell(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: replace_typical_misspell(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: replace_typical_misspell(x))
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<define_variables>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train_df['question_text'] = train_df['question_text'].str.lower()
val_df['question_text'] = val_df['question_text'].str.lower()
test_df['question_text'] = test_df['question_text'].str.lower()
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SKIP_VALIDATION = True
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<define_variables>
full_text = list(train_df['question_text'].values)+ list(val_df['question_text'].values)+list(test_df['question_text'].values )
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<categorify>
import pandas as pd
import numpy as np
from sklearn.preprocessing import OneHotEncoder
import keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU
from keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D, BatchNormalization
from keras.models import Model, load_model
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras import backend as K
from keras.engine import InputSpec, Layer
from keras.optimizers import Adam, RMSprop
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, LearningRateScheduler
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES =(1 - SKIP_VALIDATION)* count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
tokenizer = Tokenizer(lower = True, filters = '')
 tokenizer.fit_on_texts(full_text )
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model.summary()<train_model>
train_tokenized = tokenizer.texts_to_sequences(train_df['question_text'])
 val_tokenized = tokenizer.texts_to_sequences(val_df['question_text'])
 test_tokenized = tokenizer.texts_to_sequences(test_df['question_text'] )
history = model.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
max_len = 70
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_val = pad_sequences(val_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )
with strategy.scope() :
Inet = InceptionResNetV2(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model2 = tf.keras.Sequential([
Inet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()<train_model>
embedding_path = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
history2 = model2.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
embed_size = 300
max_features = 100000
with strategy.scope() :
desnet=DenseNet201(
 weights='imagenet',
include_top=False 
)
 model3 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model3.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model3.summary()<train_model>
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32' )
history3 = model3.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<predict_on_test>
def get_embed_mat(embedding_path):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m = model.predict(images_ds)
 m2 = model2.predict(images_ds)
 m3 = model3.predict(images_ds)
 scores = []
best_score,best_alpha,best_beta=0,0,0
for alpha in np.linspace(0,1,100):
for beta in np.linspace(0,1,100):
cm_probabilities = alpha*m+(beta)*m2+(1-alpha-beta)*m3
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 if(score > best_score):
best_alpha = alpha
best_beta = beta
best_score = score
scores.append(best_score)
 plt.plot(scores)
 print("best_alha:",best_alpha,"best_beta:",best_beta)
 cm_probabilities = best_alpha*m+best_beta*m2+(1-best_alpha)*m3
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 else:
best_alpha = 0.37
best_beta = 0.26 
<compute_test_metric>
y = train_df['target']
print(best_alpha,best_beta )<compute_test_metric>
y_val=val_df['target']
if not SKIP_VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<predict_on_test>
file_path = "model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 10 )
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = 0.85*(0.56*model.predict(test_images_ds)+0.44*model3.predict(test_images_ds)) +(0.15)*model2.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 np.savez_compressed('probs.npz', a=probabilities )<install_modules>
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU, CuDNNGRU, CuDNNLSTM, BatchNormalization
from keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D
from keras.models import Model, load_model
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras import backend as K
from keras.engine import InputSpec, Layer
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, TensorBoard, Callback, EarlyStopping
!pip install -q efficientnet<import_modules>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
print("Tensorflow version " + tf.__version__)
<set_options>
def build_model(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
inp = Input(shape =(max_len,))
x = Embedding(100001, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(max_len )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 history = model.fit(X_train, y, batch_size = 256, epochs = 3, validation_split=0.1,
verbose = 1, callbacks = [check_point, early_stop])
 return model
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
IMAGE_SIZE = [512, 512]
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<init_hyperparams>
embedding_matrix = get_embed_mat(embedding_path )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<define_variables>
model = build_model(lr = 1e-3, lr_d = 0, units = 128, dr = 0.5 )
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SKIP_VALIDATION = False
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<define_variables>
pred = model.predict(X_val, batch_size = 1024, verbose = 1 )
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<categorify>
predictions =(pred > 0.35 ).astype(int )
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None, figsize = 13.0):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = figsize
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )<categorify>
print(metrics.accuracy_score(y_val,predictions))
print(metrics.f1_score(y_val,predictions))
<choose_model_class><EOS> 
pred = model.predict(X_test, batch_size = 1024, verbose = 1)
 predictions =(pred > 0.37 ).astype(int)
 sub['prediction'] = predictions
sub.to_csv("submission.csv", index=False )