Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<define_variables>
!pip install efficientnet
<normalization>
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from keras.applications.densenet import DenseNet201
from sklearn import metrics
from sklearn.model_selection import train_test_split
from keras.callbacks import ModelCheckpoint
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM,self ).__init__()
self.p = Parameter(torch.ones(1)*p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(self.eps)+ ')'
class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class isupnet(nn.Module):
def __init__(self, num_classes,base_model, **kwargs):
super(isupnet, self ).__init__()
self.base_model_name=base_model
self.avg_pooling=GeM(p=3)
 self.max_pooling=nn.AdaptiveMaxPool2d(1)
 if base_model[0]=='e':
self.base = EfficientNet.from_name(base_model)
 feature = self.base._fc.in_features
else:
base_net=torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext50_32x4d_ssl')
 feature = base_net.fc.in_features
self.base=nn.Sequential(*list(base_net.children())[:-2])
 self.fc = nn.Linear(in_features=2*feature,out_features=feature,bias=True)
 self.classifiers = nn.Linear(in_features=feature, out_features=num_classes)
 self.sigmoid = MemoryEfficientSwish()
self.dropout=nn.Dropout(0.4)
 self.bn=nn.BatchNorm1d(feature)
 def freeze_base(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze_all(self):
for p in self.parameters() :
p.requires_grad = True
def forward(self, x1):
bs=x1.size(0)
 if self.base_model_name[0]=='e':
x2 = self.base.extract_features(x1)
 feat1=self.avg_pooling(x2)
 feat2=self.max_pooling(x2)
 feat1=feat1.view(bs,-1)
 feat2=feat2.view(bs,-1)
 feat=torch.cat([feat1,feat2],1)
 feat = self.fc(feat)
 feat = self.dropout(feat)
 ys = self.classifiers(feat)
 else:

 x=self.base(x1)
 feat1=self.avg_pooling(x)
 feat2=self.max_pooling(x)
 feat1=feat1.view(bs,-1)
 feat2=feat2.view(bs,-1)
 feat=torch.cat([feat1,feat2],1)
 feat = self.fc(feat)
 feat = self.dropout(feat)
 ys = self.classifiers(feat)
 return ys<normalization>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
if __name__ == '__main__':
use_gpu = torch.cuda.is_available()
if use_gpu:
cudnn.benchmark = True
torch.cuda.manual_seed_all(0)
 else:
print("Currently using CPU(GPU is highly recommended)")
 transform2 = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
 test_data=panda_dataset(content,transform2)
 net = EfficientNet.from_name('efficientnet-b0')
 feature = net._fc.in_features
net._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net._avg_pooling=GeM()
net1 = EfficientNet.from_name('efficientnet-b0')
 feature = net1._fc.in_features
net1._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net1._avg_pooling=GeM()
net2 = EfficientNet.from_name('efficientnet-b0')
 feature = net2._fc.in_features
net2._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net2._avg_pooling=GeM()
net3 = EfficientNet.from_name('efficientnet-b0')
 feature = net3._fc.in_features
net3._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net3._avg_pooling=GeM()
net4 = EfficientNet.from_name('efficientnet-b0')
 feature = net4._fc.in_features
net4._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net4._avg_pooling=GeM()
net5 = EfficientNet.from_name('efficientnet-b0')
 feature = net5._fc.in_features
net5._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net5._avg_pooling=GeM()
net6 = EfficientNet.from_name('efficientnet-b0')
 feature = net6._fc.in_features
net6._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net6._avg_pooling=GeM()
net7 = EfficientNet.from_name('efficientnet-b0')
 feature = net7._fc.in_features
net7._fc = nn.Linear(in_features=feature,out_features=6,bias=True)
 net7._avg_pooling=GeM()
nets=[net,net1,net2,net3,net4,net5,net6,net7]

 if use_gpu:
net=net.cuda()
net1=net1.cuda()
net2=net2.cuda()
net3=net3.cuda()
net4=net4.cuda()
net5=net5.cuda()
net6=net6.cuda()
net7=net7.cuda()
net.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_combine_efficientnet-b0_0.00032_adam_4_bce_karmax_gem_1536_augt_845_kar8731.pkl'))
net1.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_36_256_256_efficientnet-b0_0.0003_adam_3_bce_maxest_aug_gem_8624.pkl'))
net2.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_36_256_256_efficientnet-b0_0.0003_adam_1_bce_maxest_gem_8647.pkl'))
net3.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_36_256_256_efficientnet-b0_0.00032_adam_4_bce_maxest_aug_gem_8616.pkl'))
net4.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_36_256_256_efficientnet-b0_0.00032_adam_4_bce_maxest_augb_gem_867.pkl'))
net5.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_256_efficientnet-b0_0.00032_adam_4_bce_karmax_gem_1536_augb_kar8843_8647.pkl'))
net6.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_combine_efficientnet-b0_0.00032_adam_4_bce_karmax_gem_1536_naugt_nbal_kar8928_8725.pkl'))
net7.load_state_dict(load_para_dict('/kaggle/input/pandamodels/model_256_efficientnet-b0_0.00032_adam_0_bce_karmax_gem_1536_augb_kar8741_8567.pkl'))
print(net._avg_pooling.p)
 dataloader_test=DataLoader(
 test_data,batch_size=1, shuffle = False, num_workers= 5)
 idx=0
max_correct=0
with open('/kaggle/working/submission.csv',"a+", newline='')as f:
f_csv = csv.writer(f)
 f_csv.writerow(['image_id','isup_grade'])
 with torch.no_grad() :
net.eval()
net1.eval()
net2.eval()
net3.eval()
net4.eval()
net5.eval()
net6.eval()
net7.eval()
total=0
total_loss=0
correct=0
for id,item in enumerate(dataloader_test):
print('id ',id)
 name,datas,datas_com=item
if use_gpu:
for i in range(len(datas)) :
datas[i]=datas[i].cuda()
datas_com[i]=datas_com[i].cuda()
print(datas[0].size())
 outs=[]
for i in range(len(nets)) :
if i==0 or i== 6:
for j in range(len(datas_com)) :
outs.append(nets[i](datas_com[j]))
else:
for j in range(len(datas_com)) :
outs.append(nets[i](datas[j]))
out=outs[0]
for i in range(1,len(outs)) :
out=out+outs[i]
out=out/len(outs)
 print(out)
 predicted=get_preds(( torch.sigmoid(out)> 0.5 ).cpu().numpy())
 hh=[str(name[0]),str(predicted[0])]
print(hh)
 with open('/kaggle/working/submission.csv',"a+", newline='')as f:
f_csv = csv.writer(f)
 f_csv.writerow(hh)
 print(pd.read_csv('/kaggle/working/submission.csv' ).isup_grade.value_counts())
<import_modules>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IM_Z = 768
import re
import time
import gc
import random
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from tabulate import tabulate
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data<load_pretrained>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
if not os.path.exists('./embeddings'):
os.mkdir('./embeddings')
 os.mkdir('./embeddings/glove.840B.300d/')
 with zipfile.ZipFile('.. /input/quora-insincere-questions-classification/embeddings.zip', 'r')as z:
with z.open('glove.840B.300d/glove.840B.300d.txt')as zf, open('./embeddings/glove.840B.300d/glove.840B.300d.txt', 'wb')as f:
shutil.copyfileobj(zf, f )<init_hyperparams>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.1, random_state=2020 )
embed_size = 300
max_features = 30000
maxlen = 72
batch_size = 512
train_epochs = 5
SEED = 1029
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
def decode_image(filename, label=None, image_size=(IM_Z, IM_Z)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
x = re.sub('[0-9]{1}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will",
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am",
"I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have",
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will",
"she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have",
"so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would",
"they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are",
"you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are',
'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',
'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist',
'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what',
'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 spaces_re = re.compile('\s')
 number_re, numbers_re = re.compile('(?<![0-9])[0-9]{1}(?![0-9])'), re.compile('[0-9]{2,}')
 CAPS_re = re.compile('[A-Z]{2,}')
 punct_re = re.compile('(\?|!){2,}')
 def prop_pattern(text, pattern_re):
return len(pattern_re.findall(text)) /(len(spaces_re.findall(text)) +1)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<load_from_csv>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv" )<count_values>
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
ch_p = ModelCheckpoint(filepath="model_ef.h5", monitor='val_loss', save_weights_only=True,
verbose=1 )
description = train_df['target'].value_counts(sort=False)
 print(tabulate([['Count',description[0],description[1]],
['Proportion',"{0:.0%}".format(description[0]/sum(description)) ,"{0:.0%}".format(description[1]/sum(description)) ]],
headers=['',description.index[0], description.index[1]]))
del description<feature_engineering>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(IM_Z, IM_Z, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
words_nb_df = train_df.copy()
words_nb_df['word_nb'] = words_nb_df["question_text"].apply(lambda x : len(spaces_re.findall(x)))
 print(tabulate([['Mean',words_nb_df[words_nb_df['target']==0]['word_nb'].mean() ,words_nb_df[words_nb_df['target']==1]['word_nb'].mean() ],
['25%',words_nb_df[words_nb_df['target']==0]['word_nb'].quantile(0.25),words_nb_df[words_nb_df['target']==1]['word_nb'].quantile(0.25)],
['Med',words_nb_df[words_nb_df['target']==0]['word_nb'].median() ,words_nb_df[words_nb_df['target']==1]['word_nb'].median() ],
['75%',words_nb_df[words_nb_df['target']==0]['word_nb'].quantile(0.75),words_nb_df[words_nb_df['target']==1]['word_nb'].quantile(0.75)]],
headers=['','Sincere', 'Insincere']))
del words_nb_df<define_variables>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
pattern_list = [number_re, numbers_re, CAPS_re, punct_re, mispellings_re]
legends = ["Number", "Numbers", "CAPS", "Punctuation", "Mispellings"]
display = []
sincere_df = train_df[train_df['target']==0]["question_text"].copy()
insincere_df = train_df[train_df['target']==1]["question_text"].copy()
for i in range(len(legends)) :
if legends[i] == "Mispellings" :
sincere_df = sincere_df.apply(lambda x: x.lower())
 insincere_df = insincere_df.apply(lambda x: x.lower())
 metrics_sincere = sincere_df.apply(lambda x: prop_pattern(x, pattern_list[i]))
metrics_insincere = insincere_df.apply(lambda x: prop_pattern(x, pattern_list[i]))
display.append([legends[i] + " - Prop(Words)", "{0:.1%}".format(metrics_sincere.mean()),"{0:.1%}".format(metrics_insincere.mean())])
 display.append([legends[i] + " - Prop(Texts)", "{0:.1%}".format(sum(metrics_sincere>0)/len(metrics_sincere)) ,"{0:.1%}".format(sum(metrics_insincere>0)/len(metrics_insincere)) ])
 print(tabulate(display, headers=['','Sincere', 'Insincere']))
del sincere_df, insincere_df, metrics_sincere, metrics_insincere, train_df, test_df<load_from_csv>
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule, ch_p],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
def load_and_prec() :
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index<load_pretrained>
with strategy.scope() :
model1 = tf.keras.Sequential([
tf.keras.applications.DenseNet201(
 input_shape=(IM_Z, IM_Z, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model1.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
tqdm.pandas()
if not os.path.exists('./train_X.npy'):
start_time = time.time()
train_X, test_X, train_y, word_index = load_and_prec()
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
np.save('./train_X', train_X, allow_pickle=True)
 np.save('./test_X', test_X, allow_pickle=True)
 np.save('./train_y', train_y, allow_pickle=True)
 with open('./word_index.pickle', 'wb')as handle:
pickle.dump(word_index, handle, protocol=pickle.HIGHEST_PROTOCOL)
 else :
train_X = np.load('./train_X.npy', allow_pickle=True)
 test_X = np.load('./test_X.npy', allow_pickle=True)
 train_y = np.load('./train_y.npy', allow_pickle=True)
 with open('./word_index.pickle', 'rb')as handle:
tokenizer = pickle.load(handle)
 gc.collect()<statistical_test>
ch_p_den = ModelCheckpoint(filepath="model_den.h5", monitor='val_loss', save_weights_only=True,
verbose=1 )
def load_glove(word_index):
EMBEDDING_FILE = '.. /working/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index)+1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_pretrained>
history1 = model1.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule, ch_p_den],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
<choose_model_class><EOS> 
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()