Kaggle competition: SIIM-ISIC Melanoma Classification
train_df['target_relabeled'] = train_df['target'].copy()<feature_engineering>
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 16,
read_size = 256,
crop_size = 250,
net_size = 384,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 5,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 13,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)
train_df.loc[train_df['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0
train_df.loc[train_df['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you!
train_df.loc[train_df['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0
train_df.loc[train_df['text'] == '.POTUS
train_df.loc[train_df['text'] == 'CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1
train_df.loc[train_df['text'] == '
train_df.loc[train_df['text'] == 'In
train_df.loc[train_df['text'] == 'Who is bringing the tornadoes and floods.Who is bringing the climate change.God is after America He is plaguing her
train_df.loc[train_df['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B.Cooper.http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1
train_df.loc[train_df['text'] == "Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "wowo--=== 12000 Nigerian refugees repatriated from Cameroon", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "He came to a land which was engulfed in tribal war and turned it into a land of peace i.e.Madinah.
 train_df.loc[train_df['text'] == "Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it
train_df.loc[train_df['text'] == "The Prophet(peace be upon him)said 'Save yourself from Hellfire even if it is by giving half a date in charity.'", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "Caution: breathing may be hazardous to your health.", 'target_relabeled'] = 1
train_df.loc[train_df['text'] == "I Pledge Allegiance To The P.O.P.E.And The Burning Buildings of Epic City.??????", 'target_relabeled'] = 0
train_df.loc[train_df['text'] == "
train_df.loc[train_df['text'] == "that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time", 'target_relabeled'] = 0<import_modules>
!pip install -q efficientnet
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import StratifiedShuffleSplit<choose_model_class>
seed=123
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
K = 5
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True )<count_unique_values>
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
DISASTER = train_df['target'] == 1
print('Training set shape: ', train_df.shape)
 print('Training set keyword count: ', train_df['keyword'].nunique() )<train_model>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
for fold,(train_idx, val_idx)in enumerate(skf.split(train_df['text_cleaned'], train_df['target']), 1):
print('Fold {} training shape: {}, Validation shape: {}'.format(fold, train_df.loc[train_idx, 'text_cleaned'].shape, train_df.loc[val_idx, 'text_cleaned'].shape))<import_modules>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
import tensorflow as tf
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score<prepare_x_and_y>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
class ClassificationReport(tf.keras.callbacks.Callback):
def __init__(self, training_data=() , validation_data=()):
super(tf.keras.callbacks.Callback, self ).__init__()
self.X_train, self.y_train = training_data
self.train_precision_scores = []
self.train_recall_scores = []
self.train_f1_scores = []
self.X_val, self.y_val = validation_data
self.val_precision_scores = []
self.val_recall_scores = []
self.val_f1_scores = []
def on_epoch_end(self, epoch, logs={}):
train_predictions = np.round(self.model.predict(self.X_train, verbose=0))
val_predictions = np.round(self.model.predict(self.X_val, verbose=0))
train_precision = precision_score(self.y_train, train_predictions, average='macro')
 train_recall = recall_score(self.y_train, train_predictions, average='macro')
 train_f1 = f1_score(self.y_train, train_predictions, average='macro')
 self.train_precision_scores.append(train_precision)
 self.train_recall_scores.append(train_recall)
 self.train_f1_scores.append(train_f1)
 val_precision = precision_score(self.y_val, val_predictions, average='macro')
 val_recall = recall_score(self.y_val, val_predictions, average='macro')
 val_f1 = f1_score(self.y_val, val_predictions, average='macro')
 self.val_precision_scores.append(val_precision)
 self.val_recall_scores.append(val_recall)
 self.val_f1_scores.append(val_f1)
 print('
Epoch: {} - Train Precision: {:.6} - Train Recall: {:.6} - Train F1: {:.6}'.format(epoch+1, train_precision, train_recall, train_f1))
print('Epoch: {} - Validation Precision: {:.6} - Validation Recall: {:.6} - Validation F1: {:.6}'.format(epoch + 1, val_precision, val_recall, val_f1))<load_pretrained>
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
%%time
glove_embeddings = np.load('.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', allow_pickle=True )<string_transform>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
class DisasterDetector:
def __init__(self, max_seq_length=128, lr=0.0001, epochs=20, batch_size=128):
self.max_seq_length = max_seq_length
self.lr = lr
self.epochs = epochs
self.batch_size = batch_size
self.tokenizer = None
self.models = []
self.scores = {}
def _tokenizer(self, X):
self.tokenizer = tf.keras.preprocessing.text.Tokenizer()
self.tokenizer.fit_on_texts(X['text_cleaned'].values)
 return self.tokenizer
def encode(self, texts, targets):
input_sequences = self.tokenizer.texts_to_sequences(texts)
 padded_sequences = tf.keras.preprocessing.sequence.pad_sequences(input_sequences,
maxlen=self.max_seq_length)
 target_categorical = tf.keras.utils.to_categorical(targets, num_classes=len(set(targets)))
 return padded_sequences, target_categorical
def get_embeddings(self, glove):
embedding_matrix = np.zeros(( len(self.tokenizer.word_index)+ 1, 300))
for word, i in self.tokenizer.word_index.items() :
embedding_vector = glove.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_model(self, embedding_weights):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Input(shape=(self.max_seq_length,)))
 model.add(tf.keras.layers.Embedding(len(self.tokenizer.word_index)+1, 300, weights=[embedding_weights], trainable=False, input_length=self.max_seq_length))
model.add(tf.keras.layers.LSTM(256, recurrent_dropout=0.2))
model.add(tf.keras.layers.Dense(2, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
def train(self, X, glove_embeddings):
self.tokenizer = self._tokenizer(X)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X['text_cleaned'], X['keyword'])) :
print('
Fold ', fold)
 X_train_encoded, y_train_encoded = self.encode(X.loc[train_idx, 'text_cleaned'].str.lower() , X.loc[train_idx, 'target_relabeled'])
 X_val_encoded, y_val_encoded = self.encode(X.loc[train_idx, 'text_cleaned'].str.lower() , X.loc[train_idx, 'target_relabeled'])
 embeddings = self.get_embeddings(glove_embeddings)
 metrics = ClassificationReport(training_data=(X_train_encoded, y_train_encoded), validation_data=(X_val_encoded, y_val_encoded))
model = self.build_model(embeddings)
 model.fit(X_train_encoded, y_train_encoded,
validation_data=(X_val_encoded, y_val_encoded),
epochs=self.epochs,
callbacks=[metrics],
batch_size=self.batch_size)
 self.models.append(model)
 self.scores[fold] = {
'train': {
'precision': metrics.train_precision_scores,
'recall': metrics.train_recall_scores,
'f1': metrics.train_f1_scores
},
'validation': {
'precision': metrics.val_precision_scores,
'recall': metrics.val_recall_scores,
'f1': metrics.val_f1_scores
}
}
def predict(self, X):
X_encoded = self.tokenizer.texts_to_sequences(X)
 X_padded = tf.keras.preprocessing.sequence.pad_sequences(X_encoded, maxlen=self.max_seq_length)
 return np.argmax(self.models[0].predict(X_padded))<feature_engineering>
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, target:(prepare_image(img, cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )
disaster = DisasterDetector()<train_model>
ds = get_dataset(files_test, CFG, labeled=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
%%time
disaster.train(train_df, glove_embeddings )<save_to_csv>
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
sample_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sample_submission['target'] = test_df['text_cleaned'].apply(lambda x: disaster.predict([x]))
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()<load_from_csv>
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 outputs = []
for i in range(cfg['net_count']):
constructor = getattr(efn, f'EfficientNetB{i}')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.summary()
return model
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape: ",train.shape)
 print("Test shape: ",test.shape )<count_missing_values>
def compile_new_model(cfg):
with strategy.scope() :
model = get_model(cfg)
 losses = [tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac'])
 for i in range(cfg['net_count'])]
model.compile(
 optimizer = cfg['optimizer'],
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model
train.isnull().sum()<count_missing_values>
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 ds_train = ds_train.map(lambda img, label:(img, tuple([label] * CFG['net_count'])))
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS)
 model = compile_new_model(CFG)
 history = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)] )
test.isnull().sum()<define_variables>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = np.stack(probs)
 probs = probs[:,:cnt_test * CFG['tta_steps']]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=1), axis=1)
 probs = np.mean(probs, axis=1 )
datostotales = [train, test]<feature_engineering>
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
for datatotal in datostotales:
datatotal['Title'] = datatotal['Name'].str.extract('([A-Za-z]+)\.', expand=False )<categorify>
for i in range(CFG["net_count"]):
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[i,:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission_model_{i}.csv', index=False )
<drop_column><EOS> 
submission = pd.DataFrame(dict(
 image_name = image_names,
target = np.mean(probs[:,:,0], axis=0)))
 submission = submission.sort_values('image_name')
 submission.to_csv('submission_models_blended.csv', index=False )