Kaggle competition: Quora Insincere Questions Classification
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
warmup_history = model.fit(x=get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=get_validation_dataset() ,
epochs=WARMUP_EPOCHS,
verbose=2 ).history<choose_model_class>
df = pd.read_csv('.. /input/train.csv')
 df.columns
parser = English()
def cleaner(doc):
return ' '.join([token.lemma_ for token in parser(doc.lower())if token.lemma_ not in STOP_WORDS and token.lemma_ != "-PRON-" and token.lemma_ not in punctuation])
 tqdm.pandas()
df['clean'] = df['question_text'].progress_apply(cleaner )
LR_START = 0.00000001
LR_MIN = 0.000001
LR_MAX = LEARNING_RATE
LR_RAMPUP_EPOCHS = 3
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
sns.set(style='whitegrid')
 fig, ax = plt.subplots(figsize =(20, 6))
plt.plot(rng, y)
 print("Learning rate schedule : {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def cleaner_2(doc):
doc = str(doc)
 doc = re.sub('[0-9]{5,}', '
doc = re.sub('[0-9]{4}', '
doc = re.sub('[0-9]{3}', '
doc = re.sub('[0-9]{2}', '
for punct in puncts:
doc = doc.replace(punct, f' {punct} ')
 return doc
tqdm.pandas()
df['clean_2'] = df['question_text'].progress_apply(cleaner_2 )
for layer in model.layers:
layer.trainable = True
checkpoint = ModelCheckpoint(model_path, monitor = 'val_loss', mode='min', save_best_only=True)
 es = EarlyStopping(monitor = 'val_loss', mode='min', patience=ES_PATIENCE,
restore_best_weights = True, verbose=1)
 lr_callback = LearningRateScheduler(lrfn, verbose=1)
 callback_list = [checkpoint, es, lr_callback]
optimizer = optimizers.Adam(lr=LEARNING_RATE)
 model.compile(optimizer = optimizer,
loss = 'sparse_categorical_crossentropy',
metrics = metric_list)
 model.summary()<train_model>
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'cant': 'cannot',
'couldnt': 'could not',
'shant':'shall not',
'shouldnt': 'should not',
'wont': 'will not',
'wouldnt': 'would not',
'havent': 'have not',
'hadnt':'had not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
history = model.fit(
 x = get_training_dataset() ,
steps_per_epoch = STEPS_PER_EPOCH,
validation_data = get_validation_dataset() ,
callbacks = callback_list,
epochs = EPOCHS,
verbose = 2
).history<predict_on_test>
tick = time.time()
brown_vocab_lower = brown.words(categories=brown.categories())
 brown_vocab_lower = set(map(lambda x:x.lower() ,brown_vocab_lower))
print("brown vocab generated in {} seconds.".format(str(time.time() -tick)))
 tick = time.time()
EMBEDDING_FILE = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
def get_coefs(word,*arr): return word
glove_vocab = set(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8'))
glove_vocab = set(map(lambda x:x.lower() ,glove_vocab))
print("glove vocab generated in {} seconds.".format(str(time.time() -tick)))
 tick = time.time()
EMBEDDING_FILE = ".. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec"
def get_coefs(word,*arr): return word
wiki_vocab = set(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8'))
wiki_vocab = set(map(lambda x:x.lower() ,wiki_vocab))
print("wiki vocab generated in {} seconds.".format(str(time.time() -tick)))
 tick = time.time()
EMBEDDING_FILE = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
def get_coefs(word,*arr): return word
paragram_vocab = set(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8', errors='ignore')if len(o)>100)
 paragram_vocab = set(map(lambda x:x.lower() ,paragram_vocab))
print("paragram vocab generated in {} seconds.".format(str(time.time() -tick)) )
x_train = train_dataset.map(lambda image, label : image)
 train_preds = model.predict(x_train)
 train_preds = np.argmax(train_preds, axis=-1)
 print(classification_report(y_train, train_preds, target_names=CLASSES))<predict_on_test>
final_vocab = set(list(glove_vocab)+ list(wiki_vocab)+ list(brown_vocab_lower)+ list(paragram_vocab))
del glove_vocab,wiki_vocab,brown_vocab_lower,paragram_vocab, EMBEDDING_FILE
gc.collect()
x_valid = valid_dataset.map(lambda image, label : image)
 valid_preds = model.predict(x_valid)
 valid_preds = np.argmax(valid_preds, axis=-1)
 print(classification_report(y_valid, valid_preds, target_names=CLASSES))<predict_on_test>
def get_feats(doc):
token_length = len(doc.split())
 i1 = len(re.findall(r'(?=\bI\b|\bwe\b|\bme\b|\bus\b|\bmy\b|\bour\b|\bmine\b|\bours\b)', doc)) /token_length
i2 = len(re.findall(r'(?=\byou\b|\byour\b|\byours\b)', doc)) /token_length
i3 = len(re.findall(r'(?=\bhe\b|\bshe\b|\bit\b|\bhim\b|\bher\b|\bhis\b|\bhers\b|\bits\b)', doc)) /token_length
i4 = len(re.findall(r'(?=\bthey\b|\bthem\b|\btheir\b|\btheirs\b)', doc)) /token_length
i5 = len(re.findall(r'(?=\bwhat\b|\bwhatever\b|\bwhich\b|\bwhichever\b|\bwho\b|\bwhoever\b|\bwhom\b|\bwhomever\b|\bwhoses\b)', doc))
i6 = len(re.findall(r'(?=\bmyself\b|\byourself\b|\bherself\b|\bhimself\b|\bitself\b|\bourselves\b|\byourselves\b|\bthemselves\b)', doc))
i7 = len(re.findall(r'(?=\bfor\b|\band\b|\bnor\b|\bbut\b|\byet\b|\bso\b|\bbefore\b|\bonce\b|\bsince\b|\bthough\b|\bwhile\b|\bas\b|\bbecause\b|\bafter\b)', doc)) /token_length
i8 =(len(doc)- len(re.findall('[a-zA-Z]', doc)) - doc.count(' ')- len(re.findall('[0-9]', doc)) - doc.count(',')- doc.count('?')- doc.count('.'))
i9 = doc.count(',')
 i10 = doc.count('?')
 i11 = token_length
i12 = len(re.findall(r'(?=\bcan't\b|\bcouldn't\b|\bshan't\b|\bshouldn't\b|\bwouldn't\b|\bhaven't\b|\bdidn't\b|\bnot\b|\bnever\b|\bwon't\b|\bdon't\b|\bhadn't\b|\bcant\b|\bcouldnt\b|\bshant\b|\bshouldnt\b|\bwouldnt\b|\bhavent\b|\bdidnt\b|\bwont\b|\bdont\b|\bhadnt\b)', doc))
temp_doc =(" " ).join(re.findall(r"[a-zA-Z0-9']+", doc))
i13 = len(set(temp_doc.split())-final_vocab)
 return(i1,i2,i3,i4,i5,i6,i7,i8,i9,i10,i11,i12,i13 )
x_train_samp, y_train_samp = dataset_to_numpy_util(train_dataset, 9)
 train_samp_preds = model.predict(x_train_samp, batch_size=9)
 display_9_images_with_predictions(x_train_samp, train_samp_preds, y_train_samp )<predict_on_test>
df['fp'] = df['feat_vect'].apply(lambda x: x[0])
 df['sp'] = df['feat_vect'].apply(lambda x: x[1])
 df['tps'] = df['feat_vect'].apply(lambda x: x[2])
 df['tpp'] = df['feat_vect'].apply(lambda x: x[3])
 df['conjunction'] = df['feat_vect'].apply(lambda x: x[6])
 df['special_chars'] = df['feat_vect'].apply(lambda x: x[7])
 df['commas'] = df['feat_vect'].apply(lambda x: x[8])
 df['qm'] = df['feat_vect'].apply(lambda x: x[9])
 df['len'] = df['feat_vect'].apply(lambda x: x[10])
 df['sm'] = df['feat_vect'].apply(lambda x: x[12])
 sincere = df[df['target']==0]
insincere = df[df['target']==1]
x_valid_samp, y_valid_samp = dataset_to_numpy_util(valid_dataset, 9)
 valid_samp_preds = model.predict(x_valid_samp, batch_size=9)
 display_9_images_with_predictions(x_valid_samp, valid_samp_preds, y_valid_samp )<predict_on_test>
test = pd.read_csv('.. /input/test.csv')
 test['clean'] = test['question_text'].apply(cleaner )
x_test = test_dataset.map(lambda image, idnum:image)
 test_preds = model.predict(x_test)
 test_preds = np.argmax(test_preds, axis=-1 )<save_to_csv>
test['clean_2'] = test['question_text'].apply(cleaner_2 )
test_ids_ds = test_dataset.map(lambda image, idnum:idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 submission = pd.DataFrame(test_ids, columns=['id'])
 submission['label'] = test_preds
submission.to_csv('submission.csv', index=False)
 display(submission.head(10))<import_modules>
tqdm.pandas()
test['feat_vect'] = test['question_text'].progress_apply(get_feats)
 test['fp'] = test['feat_vect'].apply(lambda x: x[0])
 test['sp'] = test['feat_vect'].apply(lambda x: x[1])
 test['tps'] = test['feat_vect'].apply(lambda x: x[2])
 test['tpp'] = test['feat_vect'].apply(lambda x: x[3])
 test['conjunction'] = test['feat_vect'].apply(lambda x: x[6])
 test['special_chars'] = test['feat_vect'].apply(lambda x: x[7])
 test['commas'] = test['feat_vect'].apply(lambda x: x[8])
 test['qm'] = test['feat_vect'].apply(lambda x: x[9])
 test['len'] = test['feat_vect'].apply(lambda x: x[10])
 test['sm'] = test['feat_vect'].apply(lambda x: x[12])
 test.head()
print("Currently using Tensorflow version " + tf.__version__ )<choose_model_class>
del final_vocab
gc.collect()
DEVICE = 'TPU'
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
def my_tweet_tokenizer(doc):
t1 = TweetTokenizer()
return t1.tokenize(doc)
 c2 = CountVectorizer(tokenizer=my_tweet_tokenizer, ngram_range=(2,3), min_df=6)
 tick = time.time()
c2.fit(insincere['clean'])
 bow2 = c2.transform(insincere['clean'])
 print('Time taken to fit CV: ' + str(time.time() -tick)+ ' seconds.')
 lda2 = LatentDirichletAllocation(n_components=8, learning_method='batch')
 tick = time.time()
lda2.fit(bow2)
 print('Time taken to fit lda: ' + str(time.time() -tick)+ ' seconds.')
GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started' )<define_variables>
tick = time.time()
new_train_df = pd.concat([sincere.sample(frac=0.8, random_state=42),insincere])
 new_train_df = new_train_df.sample(frac=1, random_state=113)
 X_train, X_dev, y_train, y_dev = train_test_split(new_train_df, new_train_df['target'], test_size=0.2, random_state=42, stratify=new_train_df['target'])
 t1 = Tokenizer()
t1.fit_on_texts(pd.concat([df['clean_2'], test['clean_2']]))
t2 = Tokenizer(filters=None, char_level=True)
 t2.fit_on_texts(pd.concat([df['clean_2'], test['clean_2']]))
print('Time taken to fit keras tokenizers : ' + str(time.time() -tick)+ ' seconds.' )
SEED = 34
IMAGE_SIZE = [512, 512]
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
FOLDS = 3
TTA = 5
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]<categorify>
tick = time.time()
X_train_tts = t1.texts_to_sequences(X_train['clean_2'])
 X_dev_tts = t1.texts_to_sequences(X_dev['clean_2'])
 test_tts = t1.texts_to_sequences(test['clean_2'])
 X_train_tts = pad_sequences(X_train_tts, maxlen=90, padding='post', truncating='post')
 X_dev_tts = pad_sequences(X_dev_tts, maxlen=90, padding='post', truncating='post')
 test_tts = pad_sequences(test_tts, maxlen=90, padding='post', truncating='post')
 X_train_cts = t2.texts_to_sequences(X_train['clean_2'])
 X_dev_cts = t2.texts_to_sequences(X_dev['clean_2'])
 test_cts = t2.texts_to_sequences(test['clean_2'])
 X_train_cts = pad_sequences(X_train_cts, maxlen=180, padding='post', truncating='post')
 X_dev_cts = pad_sequences(X_dev_cts, maxlen=180, padding='post', truncating='post')
 test_cts = pad_sequences(test_cts, maxlen=180, padding='post', truncating='post')
 print('Time taken for tts conversion : ' + str(time.time() -tick)+ ' seconds.' )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example, return_image_name):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum if return_image_name else 0
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<define_variables>
tick = time.time()
bow_train = c2.transform(X_train['clean'])
 ldavec_train = lda2.transform(bow_train)
 bow_dev = c2.transform(X_dev['clean'])
 ldavec_dev = lda2.transform(bow_dev)
 bow_test = c2.transform(test['clean'])
 ldavec_test = lda2.transform(bow_test)
 print('Time taken to generate lda vectors for train, dev & test : ' + str(time.time() -tick)+ ' seconds.' )
AUTO = tf.data.experimental.AUTOTUNE
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')+ tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<categorify>
feat_train = X_train[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm']].values
feat_dev = X_dev[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm']].values
feat_test = test[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm']].values
def get_train_ds(files, tta_aug = False, cutmix_aug = False, shuffle = True,
repeat = True, labeled=True, return_image_names = True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 if tta_aug:
ds = ds.map(data_augment, num_parallel_calls = AUTO)
 ds = ds.map(transform, num_parallel_calls=AUTO)
 if cutmix_aug:
ds = ds.batch(AUG_BATCH)
 ds = ds.map(mixup_and_cutmix, num_parallel_calls=AUTO)
 ds = ds.unbatch()
ds = ds.batch(BATCH_SIZE)
 ds = ds.prefetch(AUTO)
 return ds
def get_val_ds(files, shuffle = True, labeled=True, return_image_names=False):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.batch(BATCH_SIZE)
 ds = ds.map(onehot, num_parallel_calls=AUTO)
 ds = ds.prefetch(AUTO)
 return ds 
<define_variables>
tick = time.time()
embedding_dim = 300
vocab_size = len(t1.word_index)+1
EMBEDDING_FILE = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_glove = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8'))
embedding_matrix = np.random.randn(vocab_size,embedding_dim ).astype(np.float32)* np.sqrt(2.0/vocab_size)
 for word, i in t1.word_index.items() :
embedding_vector = embeddings_glove.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
del embeddings_glove, EMBEDDING_FILE
gc.collect()
print("Embedding Matrix generated in {} seconds.".format(str(time.time() -tick)) )
classes = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<categorify>
tick = time.time()
EMBEDDING_FILE = ".. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_wiki = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8'))
embedding_matrix_wiki = np.random.randn(vocab_size,embedding_dim ).astype(np.float32)* np.sqrt(2.0/vocab_size)
 for word, i in t1.word_index.items() :
embedding_vector = embeddings_wiki.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
del embeddings_wiki, EMBEDDING_FILE
gc.collect()
print("Embedding Matrix WIKI generated in {} seconds.".format(str(time.time() -tick)) )
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return classes[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(classes[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
classes[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):
images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else classes[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()<train_model>
tick = time.time()
embeddings_google = KeyedVectors.load_word2vec_format('.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)
 print("Load time {} seconds.".format(str(time.time() -tick)) )
training_dataset = get_train_ds(TRAINING_FILENAMES, cutmix_aug = False, tta_aug = False, labeled = True,
shuffle = True, repeat = True)
 training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )<define_variables>
embedding_matrix_google = np.random.randn(vocab_size,embedding_dim ).astype(np.float32)* np.sqrt(2.0/vocab_size )
test_dataset = get_train_ds(TEST_FILENAMES, labeled = False, shuffle = True, repeat = False)
 test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )<define_variables>
tick = time.time()
for word, i in t1.word_index.items() :
try:
embedding_vector = 'init'
embedding_vector = embeddings_google[word]
embedding_matrix_google[i] = embedding_vector
except:
if embedding_vector=='init':
pass
print("Embedding Matrix google generated in {} seconds.".format(str(time.time() -tick)) )
display_batch_of_images(next(train_batch)) 
<define_variables>
del embeddings_google
gc.collect()
display_batch_of_images(next(test_batch)) 
<normalization>
mean_emb = np.mean([embedding_matrix, embedding_matrix_wiki, embedding_matrix_google], axis=0 )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))<prepare_x_and_y>
char_embedding_matrix = np.random.randn(len(t2.word_index)+1,128 ).astype(np.float32)* np.sqrt(2.0/len(t2.word_index)+1 )
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<categorify>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )
def onehot(image,label):
CLASSES = len(classes)
 return image,tf.one_hot(label,CLASSES )<categorify>
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def mixup(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = 104
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2<define_variables>
class Configuration:
def get_config_object() :
config = Configuration()
config.max_seq_size = 90
config.word_vocab_size = vocab_size
config.embedding_dim = 300
config.units1 =64
config.units2 = 32
config.units3 = 32
return config
test_dataset = get_train_ds(TEST_FILENAMES, cutmix_aug = False, tta_aug = False, labeled = False,
shuffle = True, repeat = False)
 test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )<prepare_x_and_y>
class Model:
def __init__(self, config):
self.config = config
def get_model(self):
input_layer = Input(batch_shape=(None,self.config.max_seq_size), name='input_layer')
 input_layer_2 = Input(batch_shape=(None,180), name='input_layer_cts')
 input_layer_3 = Input(batch_shape=(None,10), name='input_layer_feat')
 lda_input = Input(batch_shape=(None, 8), name='lda_input_layer')
 embed_op = Embedding(input_dim= self.config.word_vocab_size, input_length=self.config.max_seq_size, output_dim=self.config.embedding_dim,
weights=[mean_emb], trainable=True, name='word_embeddings' )(input_layer)
 spdropout = SpatialDropout1D(0.2, name='sp_dropout' )(embed_op)
 bigru1 = Bidirectional(CuDNNLSTM(units=self.config.units1, return_sequences=True, name='bigru1'))(spdropout)
 embed_op_2 = Embedding(input_dim=len(t2.word_index)+1, input_length=180, output_dim=128, name='char_embeddings' )(input_layer_2)
 spdropout_2 = SpatialDropout1D(0.2, name='sp_dropout_2' )(embed_op_2)
 conv1 = Conv1D(filters=32, kernel_size=2, name='conv1' )(spdropout_2)
 maxpool1 = GlobalMaxPool1D()(conv1)
 conv2 = Conv1D(filters=32, kernel_size=4, name='conv2' )(spdropout_2)
 maxpool2 = GlobalMaxPool1D()(conv2)
 conv3 = Conv1D(filters=32, kernel_size=8, name='conv3' )(spdropout_2)
 maxpool3 = GlobalMaxPool1D()(conv3)
 tile_layer = RepeatVector(n=90, name='tile_layer')
 tiled_op = tile_layer(lda_input)
 concat_1 = Concatenate(axis=2, name='concat_1' )([bigru1,tiled_op])
 att_dense_layer = Dense(units=1, name='attention_dense_1', activation='tanh')
 att_dense_op_1 = att_dense_layer(concat_1)
 softmax_layer = Softmax(axis=1, name='softmax_over_time')
 alpha_1 = softmax_layer(att_dense_op_1)
 dot_layer = Dot(axes=1, name='dot_layer')
 context_1 = dot_layer([alpha_1, concat_1])
 context_reshape_1 = Reshape(target_shape=(136,), name='reshape_context' )(context_1)
 selector = Lambda(lambda x: x[:, -1], name='slicer')
 lt_bigru1 = selector(bigru1)
 concat = concatenate([lt_bigru1,context_reshape_1,maxpool1,maxpool2, maxpool3,lda_input, input_layer_3], name='concatenate')
 dense1 = Dense(units=self.config.units3, activation='tanh', name='dense1' )(concat)
 drop1 = GaussianDropout(0.1, name='Dropout_1_c3' )(dense1)
 bn = BatchNormalization()(drop1)
 dense2 = Dense(units=16, activation='tanh', name='dense2' )(bn)
 dense3 = Dense(units=1, activation='sigmoid', name='output' )(dense2)
 classifier = keras.models.Model(inputs=[input_layer,input_layer_2,input_layer_3,lda_input], outputs=dense3)
 return classifier
def get_model_2(self):
input_layer = Input(batch_shape=(None,self.config.max_seq_size), name='input_layer')
 input_layer_2 = Input(batch_shape=(None,180), name='input_layer_cts')
 input_layer_3 = Input(batch_shape=(None,10), name='input_layer_feat')
 lda_input = Input(batch_shape=(None, 8), name='lda_input_layer')
 embed_op1 = Embedding(input_dim= self.config.word_vocab_size, input_length=90, output_dim=self.config.embedding_dim,
weights=[embedding_matrix], trainable=True, name='word_embeddings' )(input_layer)
 spdropout1 = SpatialDropout1D(0.2, name='sp_dropout1' )(embed_op1)
 conv1_word = Conv1D(filters=32, kernel_size=2, name='conv1_word' )(spdropout1)
 maxpool1_word = GlobalMaxPool1D()(conv1_word)
 avgpool1_word = GlobalAvgPool1D()(conv1_word)
 conv2_word = Conv1D(filters=32, kernel_size=4, name='conv2_word' )(spdropout1)
 maxpool2_word = GlobalMaxPool1D()(conv2_word)
 avgpool2_word = GlobalAvgPool1D()(conv2_word)
 conv3_word = Conv1D(filters=32, kernel_size=8, name='conv3_word' )(spdropout1)
 maxpool3_word = GlobalMaxPool1D()(conv3_word)
 avgpool3_word = GlobalAvgPool1D()(conv3_word)
 embed_op_2 = Embedding(input_dim=len(t2.word_index)+1, input_length=180, output_dim=128, name='char_embeddings' )(input_layer_2)
 spdropout_2 = SpatialDropout1D(0.2, name='sp_dropout_2' )(embed_op_2)
 conv1 = Conv1D(filters=32, kernel_size=2, name='conv1' )(spdropout_2)
 maxpool1 = GlobalMaxPool1D()(conv1)
 avgpool1 = GlobalAvgPool1D()(conv1)
 conv2 = Conv1D(filters=32, kernel_size=4, name='conv2' )(spdropout_2)
 maxpool2 = GlobalMaxPool1D()(conv2)
 avgpool2 = GlobalAvgPool1D()(conv2)
 conv3 = Conv1D(filters=32, kernel_size=8, name='conv3' )(spdropout_2)
 maxpool3 = GlobalMaxPool1D()(conv3)
 avgpool3 = GlobalAvgPool1D()(conv3)
 concat = concatenate([maxpool1_word,maxpool2_word, maxpool3_word,avgpool1_word, avgpool2_word, avgpool3_word,maxpool1,maxpool2, maxpool3,avgpool1, avgpool2, avgpool3,lda_input, input_layer_3], name='concatenate')
 dense1 = Dense(units=self.config.units3, activation='tanh', name='dense1' )(concat)
 drop1 = GaussianDropout(0.1, name='Dropout_1_c3' )(dense1)
 bn = BatchNormalization()(drop1)
 dense2 = Dense(units=16, activation='tanh', name='dense2' )(bn)
 dense3 = Dense(units=1, activation='sigmoid', name='output' )(dense2)
 classifier = keras.models.Model(inputs=[input_layer,input_layer_2,input_layer_3,lda_input], outputs=dense3)
 return classifier
def cutmix(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = 104
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2<categorify>
config = Configuration.get_config_object()
model = Model(config)
 opt = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True )
def mixup_and_cutmix(image,label):
CLASSES = len(classes)
 DIM = IMAGE_SIZE[0]
SWITCH = 1/2
CUTMIX_PROB = 2/3
MIXUP_PROB = 2/3
image2, label2 = cutmix(image, label, CUTMIX_PROB)
 image3, label3 = mixup(image, label, MIXUP_PROB)
 imgs = []; labs = []
for j in range(BATCH_SIZE):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(BATCH_SIZE,CLASSES))
return image4,label4<init_hyperparams>
classifier = model.get_model()
classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', precision, recall, f1])
 classifier.summary()
EPOCHS = 20
STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES)// BATCH_SIZE
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_DECAY =.8
def lr_schedule(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lr_schedule, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lr_schedule(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
earlystopping = EarlyStopping(monitor='val_f1', patience=1, mode='max')
 modelcheckpoint = ModelCheckpoint(filepath='model.h5', monitor='val_f1', verbose=1, save_best_only=True, mode='max')
 pbar = TQDMNotebookCallback(leave_inner=True, leave_outer=True)
 DATA_SPLIT_SEED = 2018
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300, mode='exp_range',
gamma=0.99994)
 callbacks1 = [earlystopping,modelcheckpoint, clr]
!pip install -q efficientnet
def get_DenseNet201() :
CLASSES = len(classes)
 with strategy.scope() :
dnet = DenseNet201(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 dnet.trainable = True
model = tf.keras.Sequential([
dnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_Xception() :
CLASSES = len(classes)
 with strategy.scope() :
xception = Xception(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 xception.trainable = True
model = tf.keras.Sequential([
xception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_InceptionV3() :
CLASSES = len(classes)
 with strategy.scope() :
inception = InceptionV3(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception.trainable = True
model = tf.keras.Sequential([
inception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_EfficientNetB4() :
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB4(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_EfficientNetB5() :
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB5(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_EfficientNetB6() :
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB6(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_EfficientNetB7() :
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB7(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_InceptionResNetV2() :
CLASSES = len(classes)
 with strategy.scope() :
inception_res = InceptionResNetV2(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception_res.trainable = True
model = tf.keras.Sequential([
inception_res,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model<train_model>
cw = {0:2, 1: 2.6}
classifier.fit([X_train_tts,X_train_cts,feat_train ,ldavec_train], y_train, callbacks=callbacks1 ,validation_data=([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], y_dev), epochs=1, batch_size=128, verbose=1, class_weight=cw )
histories = []
models = []
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for f,(train_index, val_index)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('-'*25)
 print(f"Training fold {f + 1} with EfficientNetB6")
 print('-'*25)
 print('Getting training data...')
 print('')
 train_ds = get_train_ds(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[train_index]['TRAINING_FILENAMES']),
cutmix_aug = True, tta_aug = False, labeled = True, shuffle = True, repeat = True)
 print('Geting validation data...')
 print('')
 val_ds = get_val_ds(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_index]['TRAINING_FILENAMES']),
labeled = True, shuffle = True, return_image_names = False)
 model = get_EfficientNetB6()
history = model.fit(train_ds,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback],
validation_data = val_ds,
verbose = 2)
 models.append(model)
 histories.append(history )<categorify>
cw = {0:1, 1: 1.6}
classifier.fit([X_train_tts,X_train_cts,feat_train ,ldavec_train], y_train, callbacks=callbacks1 ,validation_data=([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], y_dev), epochs=1, batch_size=256, verbose=1, class_weight=cw )
def get_test_dataset(filenames, shuffle = False, repeat = True, labeled = False, tta_aug = True, return_image_names = True):
ds = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 if tta_aug:
ds = ds.map(data_augment, num_parallel_calls = AUTO)
 ds = ds.map(transform, num_parallel_calls=AUTO)
 ds = ds.batch(BATCH_SIZE)
 ds = ds.prefetch(AUTO)
 return ds<groupby>
pred = classifier.predict([test_tts,test_cts,feat_test,ldavec_test], verbose=1 )
def average_tta_preds(tta_preds):
average_preds = np.zeros(( ct_test, 104))
for fold in range(FOLDS):
average_preds += tta_preds[(fold)*ct_test:(fold + 1)*ct_test] / FOLDS
return average_preds<predict_on_test>
pred =(pred > 0.5 ).astype(int )
test_ds = get_test_dataset(TEST_FILENAMES, tta_aug = True, labeled = False,
repeat = True, return_image_names = True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 ct_test = count_data_items(TEST_FILENAMES)
 STEPS = TTA * ct_test/BATCH_SIZE
print('Getting TTA predictions...')
 pred0 = models[0].predict(test_images_ds,steps = STEPS,verbose = 2)[:TTA*ct_test,]
pred1 = models[1].predict(test_images_ds,steps = STEPS,verbose = 2)[:TTA*ct_test,]
pred2 = models[2].predict(test_images_ds,steps = STEPS,verbose = 2)[:TTA*ct_test,]
print('')
 print('Averaging TTA predictions...')
 average_pred0 = average_tta_preds(pred0)
 average_pred1 = average_tta_preds(pred1)
 average_pred2 = average_tta_preds(pred1)
 print('')
 print('Merging predictions from models...')
 final_preds = np.zeros(( ct_test, 104))
final_preds += average_pred0
final_preds += average_pred1
final_preds += average_pred2
final_preds = np.argmax(final_preds, axis = 1)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 print('Done' )<prepare_output>
test['prediction'] = pred
submission = pd.DataFrame()
submission['id'] = test_ids
submission['label'] = final_preds
print(submission.shape)
 submission.head(10 )<save_to_csv>
final = test.drop(['question_text','feat_vect','fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm', 'clean','clean_2'], axis=1 )
<import_modules><EOS> 
final.to_csv('submission.csv',index=False )