Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<choose_model_class>
DEVICE = "TPU"
CFG = dict(
 batch_size = 16,
read_size = 256,
crop_size = 250,
net_size = 248,
LR_START = 0.000003,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 5,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 20,
rot = 180.0,
shr = 1.5,
hzoom = 6.0,
wzoom = 6.0,
hshift = 6.0,
wshift = 6.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)
class EnhancedLeNet(nn.Module):
def __init__(self):
super(EnhancedLeNet, self ).__init__()
self.features = nn.Sequential(
 nn.Conv2d(1, 64, 5, padding=2),
nn.ReLU(inplace=True),
nn.Conv2d(64, 64, 5, padding=2),
nn.ReLU(inplace=True),
nn.BatchNorm2d(64),
nn.MaxPool2d(2),
nn.Conv2d(64, 128, 5, padding=2),
nn.ReLU(inplace=True),
nn.Conv2d(128, 128, 5, padding=2),
nn.ReLU(inplace=True),
nn.BatchNorm2d(128),
nn.MaxPool2d(2), 
)
 self.classifier = nn.Sequential(
 nn.Linear(128*7*7, 512),
nn.ReLU(inplace=True),
nn.BatchNorm1d(512),
nn.Linear(512, 10)
)
 def forward(self, x):
x = self.features(x)
 x = x.view(x.size(0), 128*7*7)
 x = self.classifier(x)
 return x<train_on_grid>
!pip install -q efficientnet
lenet = EnhancedLeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters())
 lenet, batch_ita, loss_list, val_acc_list = training(lenet, criterion, optimizer, 30 )<load_from_csv>
random.seed(a=42)
def testing(network):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 composed_transform = transforms.Compose([Regularize() , ToTensor() ])
 digit_dataset = DigitDataset('test.csv', '.. /input/', train=False, argument=False, transform=composed_transform)
 test_dataloader = DataLoader(
 digit_dataset,
batch_size=128,
shuffle=False,
num_workers=4,
pin_memory=True 
)
 test_results = []
for i, data in enumerate(test_dataloader, 0):
digits, label = data
digits = digits.to(device)
 outputs = network(digits)
 _, predicted = torch.max(outputs, 1)
 test_results += np.int_(predicted.to("cpu" ).numpy().squeeze() ).tolist()

 test_df = pd.read_csv(".. /input/sample_submission.csv")
 assert(len(test_df)== len(test_results))
test_df.loc[:, 'Label'] = test_results
test_df.to_csv('test_results.csv', index=False)
 print("Test Results for Kaggle Generated..." )<train_on_grid>
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
lenet = EnhancedLeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters())
 lenet, batch_ita, loss_list, val_acc_list = training(lenet, criterion, optimizer, 50, test=False)
 testing(lenet )<install_modules>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
! pip install adaptnlp<define_variables>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
target_corrected = True
to_lower = True<load_from_csv>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
df_train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv', dtype={'id': np.int16, 'target': np.int8})
 df_test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv', dtype={'id': np.int16} )<string_transform>
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
def clean_tweets(tweet):

 tweet = ''.join([x for x in tweet if x in string.printable])
 tweet = re.sub(r"http\S+", "_url_", tweet)
 return tweet<define_variables>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'_emoji_', text )<categorify>
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, target:(prepare_image(img, cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )
def remove_punctuations(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text<define_variables>
ds = get_dataset(files_test, CFG, labeled=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}<categorify>
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word<categorify>
def get_model(cfg, model):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 outputs = []
constructor = getattr(efn, model)
 x = constructor(include_top=False, weights='noisy-student',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.summary()
return model
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text<feature_engineering>
def compile_new_model(cfg, model):
with strategy.scope() :
model = get_model(cfg, model)
 losses = tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac'])
 model.compile(
 optimizer = cfg['optimizer'],
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model
df_train["text"] = df_train["text"].apply(lambda x: clean_tweets(x))
df_test["text"] = df_test["text"].apply(lambda x: clean_tweets(x))
df_train["text"] = df_train["text"].apply(lambda x: remove_emoji(x))
df_test["text"] = df_test["text"].apply(lambda x: remove_emoji(x))
df_train["text"] = df_train["text"].apply(lambda x: remove_punctuations(x))
df_test["text"] = df_test["text"].apply(lambda x: remove_punctuations(x))
df_train["text"] = df_train["text"].apply(lambda x: convert_abbrev_in_text(x))
df_test["text"] = df_test["text"].apply(lambda x: convert_abbrev_in_text(x))
if to_lower:
df_train["text"] = df_train["text"].apply(lambda x: x.lower())
 df_test["text"] = df_test["text"].apply(lambda x: x.lower() )<feature_engineering>
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 ds_train = ds_train.map(lambda img, label:(img, tuple([label])))
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS)
 model_B0 = compile_new_model(CFG, 'EfficientNetB0')
 model_B3 = compile_new_model(CFG, 'EfficientNetB3')
 model_B5 = compile_new_model(CFG, 'EfficientNetB5')
 model_B6 = compile_new_model(CFG, 'EfficientNetB6')
 print("
Begin Training Models B0")
 history_B0 = model_B0.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B0
")
 Begin Training Models B1")
 Done Training model_B1
")
 Begin Training Models B2")
 Done Training model_B2
")
 print("
Begin Training Models B3")
 history_B3 = model_B3.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B3
")
 Begin Training Models B4")
 Done Training model_B4
")
 print("
Begin Training Models B5")
 history_B5 = model_B5.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B5
")
 print("
Begin Training Models B6")
 history_B6 = model_B6.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B6
")
 Begin Training Models B7")
 Done Training model_B7
" )
if target_corrected:
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
df_train.loc[df_train['id'].isin(ids_with_target_error),'target'] = 0
df_train[df_train['id'].isin(ids_with_target_error)]<import_modules>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B0 = model_B0.predict(ds_testAug, verbose=1, steps=steps )
from adaptnlp import EasyDocumentEmbeddings, SequenceClassifierTrainer<split>
train, tbd_test = train_test_split(df_train, test_size=0.2)
 test, dev = train_test_split(tbd_test, test_size=0.5 )<save_to_csv>
train.to_csv(label_training_dir + "/train.csv", index=False)
 test.to_csv(label_training_dir + "/test.csv", index=False)
 dev.to_csv(label_training_dir + "/dev.csv", index=False )<set_options>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B3 = model_B3.predict(ds_testAug, verbose=1, steps=steps )
configs = {"pool_configs": {"fine_tune_mode": "linear", "pooling": "mean"},
"rnn_configs": {"hidden_size": 512,
"rnn_layers": 1,
"reproject_words": True,
"reproject_words_dimension": 256,
"bidirectional": False,
"dropout": 0.4,
"word_dropout": 0.0,
"locked_dropout": 0.0,
"rnn_type": "GRU",
"fine_tune": True, }}<define_variables>
corpus = label_training_dir
FINETUNED_MODEL_DIR = MODEL_DIR
OUTPUT_DIR = label_training_dir
doc_embeddings = EasyDocumentEmbeddings("bert-base-uncased", methods = ["rnn"], )<choose_model_class>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B5 = model_B5.predict(ds_testAug, verbose=1, steps=steps )
sc_configs = {
"corpus": corpus,
"encoder": doc_embeddings,
"column_name_map": {3: "text", 4: "label"},
"corpus_in_memory": True,
"predictive_head": "flair",
}
sc_trainer = SequenceClassifierTrainer(**sc_configs )<init_hyperparams>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B6 = model_B6.predict(ds_testAug, verbose=1, steps=steps )
sc_lr_configs = {
"output_dir": OUTPUT_DIR,
"file_name": "learning_rate.tsv",
"start_learning_rate": 1e-8,
"end_learning_rate": 10,
"iterations": 100,
"mini_batch_size": 32,
"stop_early": True,
"smoothing_factor": 0.8,
"plot_learning_rate": True,
}
learning_rate = sc_trainer.find_learning_rate(**sc_lr_configs )<train_model>
sc_train_configs = {
"output_dir": OUTPUT_DIR,
"learning_rate": learning_rate,
"mini_batch_size": 32,
"anneal_factor": 0.5,
"patience": 2,
"max_epochs": 10,
"plot_weights": False,
"batch_growth_annealing": False,
}
sc_trainer.train(**sc_train_configs )<choose_model_class>
preds = np.stack(preds_B0)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B0 = preds.reshape(-1 )
example_text = ["that was a really bad storm! but I suppose it could have been worse"]
MODEL_PATH = OUTPUT_DIR + '/final-model.pt'
classifier = EasySequenceClassifier()
sentences = classifier.tag_text(example_text, model_name_or_path=MODEL_PATH)
 print("Label output:
")
 for sentence in sentences:
print(sentence.labels)
 print(sentence.labels[0].value)
 print(sentence.labels[0].score )<feature_engineering>
preds = np.stack(preds_B1)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B1 = preds.reshape(-1 )
df_test['target'] = df_test['text'].apply(lambda x: classifier.tag_text(x, model_name_or_path=MODEL_PATH)[0].labels[0].value )<count_values>
submission = df_test[['id', 'target']]
submission['target'].value_counts()<save_to_csv>
preds = np.stack(preds_B3)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B3 = preds.reshape(-1 )
submission.to_csv('/kaggle/working/submission.csv', index=False )<set_options>
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep' )<load_from_csv>
preds = np.stack(preds_B5)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B5 = preds.reshape(-1 )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
preds = np.stack(preds_B6)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B6 = preds.reshape(-1 )
X_train = X_train / 255.0
test = test / 255.0<categorify>
Y_train = to_categorical(Y_train, num_classes = 10 )<split>
preds =preds_B0*0.2 + preds_B3*0.3 + preds_B5*0.2 + preds_B6*0.3
random_seed = 2
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed )<choose_model_class>
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<choose_model_class><EOS> 
submission = pd.DataFrame(dict(
 image_name = image_names,
target = preds))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission.csv', index=False)
 submission.head()