Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<categorify>
import numpy as np
import pandas as pd
import os
import re
import tensorflow.keras.backend as K
train_y = to_categorical(train_label, num_classes = 10)
 print('one hot encoding vector of %d :' %train_label[3] , train_y[3] )<split>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
x_train,x_val,y_train,y_val = train_test_split(train_df,train_y,test_size = 0.2, random_state = 2)
<train_model>
df_train = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/train.csv")
 df_test = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/test.csv")
 sample_sub = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv" )
dataGenerator = ImageDataGenerator(rotation_range = 10,
width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.1)
 dataGenerator.fit(x_train )<normalization>
FOLDS = 5
IMG_SIZE = 512
BATCH_SIZE = 32
EPOCHS = 12
ExampleImg = train_df[10][:,:,0]
def translate(img,x,y):
transMat = np.float32([[1,0,x],[0,1,y]])
 shifted = cv2.warpAffine(img,transMat,(img.shape[1],img.shape[0]))
return shifted
def rotate(img,angle): 
(h,w)= img.shape[:2]
center =(w/2,h/2)
 rotatMat = cv2.getRotationMatrix2D(center,angle,1)
 rotated = cv2.warpAffine(img,rotatMat,(w,h))
return rotated
def zoom(img,scale): 
(oh,ow)= img.shape[:2]
zoomed = cv2.resize(img, None, fx = scale, fy = scale)
(nh,nw)= zoomed.shape[:2]
extend1 = int(( oh - nh)/ 2.0 + 0.5)
 extend2 = oh -(nh+extend1)
 zoomed = cv2.copyMakeBorder(zoomed,extend1,extend2,extend1,extend2,cv2.BORDER_CONSTANT,0)
 return zoomed
shifted = translate(ExampleImg,0,3)
 rotated = rotate(ExampleImg,10)
 zoomed = zoom(ExampleImg,0.8)
 plt.figure(figsize =(10,20))
plt.subplot(1,4,1)
 plt.imshow(ExampleImg,cmap = plt.cm.binary)
 plt.title('Original')
 plt.subplot(1,4,2)
 plt.imshow(rotated,cmap = plt.cm.binary)
 plt.title('rotated: 10 deg')
 plt.subplot(1,4,3)
 plt.imshow(shifted,cmap = plt.cm.binary)
 plt.title('shifted: y-3')
 plt.subplot(1,4,4)
 plt.imshow(zoomed,cmap = plt.cm.binary)
 plt.title('zoomed: 0.8 scale')
 plt.show()<choose_model_class>
GCS_PATH2 = KaggleDatasets().get_gcs_path("isic2019-512x512")
 GCS_PATH1 = KaggleDatasets().get_gcs_path("melanoma-512x512" )
def build_CNN(input_shape, output_units = 10):
input_layer = keras.layers.Input(input_shape, name = "input_layer")
 x = Conv2D(filters = 64, kernel_size =(3,3), padding = 'same' )(input_layer)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Conv2D(filters = 64, kernel_size =(3,3), padding = 'same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Conv2D(filters = 32, kernel_size =(3,3), padding = 'same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Conv2D(filters = 32, kernel_size =(3,3), padding = 'same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Flatten()(x)
 x = Dense(units = 512 )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Dense(units = 256 )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Dropout(0.25 )(x)
 x = Dense(units = 128 )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Dense(units = 64, activation = 'relu' )(x)
 output_layer = Dense(units = output_units, activation = 'softmax', name = "output_layer" )(x)
 model = keras.models.Model(inputs = [input_layer], outputs = [output_layer])
 return model<choose_model_class>
train_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')
 train_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 test_filenames = tf.io.gfile.glob(GCS_PATH1 + '/test*.tfrec')
 data_filenames = train_filenames1 + train_filenames2
batch_size = 128
epochs = 300
momentum = 0.95
lr = 5e-4
ES = keras.callbacks.EarlyStopping(monitor = "val_loss",
patience = 10,
verbose = 1)
 model = build_CNN(x_train.shape[1:],10)
 optimizer = keras.optimizers.SGD(lr = lr, momentum = 0.95)
 model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['accuracy'])
 model.summary()
history = model.fit_generator(dataGenerator.flow(x_train, y_train, batch_size = batch_size),
epochs = epochs, verbose = 2, validation_data =(x_val,y_val),
steps_per_epoch = x_train.shape[0]/batch_size )<predict_on_test>
train_filenames, valid_filenames = train_test_split(data_filenames, test_size=0.2, random_state=0, shuffle=True )
final_prediction = model.predict(test_df)
 final_prediction = np.argmax(final_prediction, axis = 1 )<save_to_csv>
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = example['image']
label = example['target']
return image, label
def read_unlabeled_tfrecord(example, return_image_name=True):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = example['image']
idnum = example['image_name']
return image, idnum if return_image_name else 0
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
submission = pd.DataFrame({'ImageId':np.arange(1,final_prediction.shape[0]+1,1),'Label':final_prediction})
 submission.to_csv('submission_v5.csv',index = False )<import_modules>
def decode_image(image_data, augment=False):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
if augment:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_saturation(image, 0, 2)
 image = tf.image.rot90(image)
 image = tf.reshape(image, [IMG_SIZE,IMG_SIZE, 3])
 return image
def get_training_dataset() :
dataset = tf.data.TFRecordDataset(train_filenames, num_parallel_reads=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(1024*8)
 dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=True), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_val_dataset() :
dataset = tf.data.TFRecordDataset(valid_filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=False), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = tf.data.TFRecordDataset(test_filenames, num_parallel_reads=AUTO)
 dataset = dataset.repeat()
dataset = dataset.map(lambda example: read_unlabeled_tfrecord(example, False), num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=False), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
import numpy as np
from tqdm import tqdm_notebook
import pandas as pd
import gc
import pdb<install_modules>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
!pip install transformers<import_modules>
def build_model(dim=512):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = efn.EfficientNetB3(input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
import plotly.graph_objs as go
from matplotlib import pyplot as plt
import plotly.offline as py
import regex as re
from bs4 import BeautifulSoup
import string
from nltk.corpus import stopwords
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.tokenize import word_tokenize
from gensim.models import word2vec<import_modules>
!pip install yapl==0.1.2 efficientnet > /dev/null
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim<load_pretrained>
input_shape =(512, 512, 3)
 def create_model() :
model = tf.keras.Sequential([
efn.EfficientNetB4(
 input_shape=input_shape,
weights='imagenet',
include_top=False 
),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 return model
nltk.download('punkt' )<set_options>
callbacks = [get_lr_callback(BATCH_SIZE)]
step_per_epoch = count_data_items(train_filenames)/BATCH_SIZE//REPLICAS
warnings.filterwarnings('ignore' )<set_options>
test_dataset = get_test_dataset(ordered=True )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<load_from_csv>
def get_dataset(filename, shuffle=False, repeat=False, return_img_name=False, augment=False, labeled=False, batch_size=32, dim=512):
dataset = tf.data.TFRecordDataset(filename, num_parallel_reads=AUTO)
 if repeat:
dataset = dataset.repeat()
if shuffle:
dataset = dataset.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
dataset = dataset.with_options(opt)
 if labeled:
dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
dataset = dataset.map(lambda example: read_unlabeled_tfrecord(example, return_img_name), num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=augment), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(batch_size * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<categorify>
FOLDS = 5
IMG_SIZE = 512
BATCH_SIZE = 32
EPOCHS = 12
weights = 1/6
def get_embedding(df,col):
corpus = create_corpus(df,col)
 w2v_model = word2vec.Word2Vec([corpus], size=50, window=10, min_count=1, workers=4)
 embedding_dict={}
for word in corpus:
embedding_dict[word] = w2v_model.wv[word]
num_words=len(corpus)
 embedding_matrix=np.zeros(( num_words,50))
for i,word in tqdm_notebook(enumerate(df[col].unique())) :
if i > num_words:
continue
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
return embedding_matrix
def create_corpus(df,col):
corpus=[]
for keyword in tqdm_notebook(df[col].unique()):
keyword=keyword.lower()
corpus.append(keyword)
 return corpus<categorify>
kfold = KFold(n_splits=FOLDS, shuffle=True, random_state=0 )
def combine_features(df,features,cat_cols):
df_copy=df.copy()
for col in cat_cols:
embedding = get_embedding(df_copy,col)
 vec = {val:embedding[i] for i, val in enumerate(df_copy[col].unique())}
for key,value in vec.items() :
df_copy[col] = df_copy[col].map(lambda x: value if x == key else x)
 embed_array = np.stack(df_copy[col].values,axis=0)
 features = np.concatenate(( features,embed_array),axis=1)
 return features<data_type_conversions>
TTA = 11
preds = np.zeros(( count_data_items(test_filenames),1))
for fold,(train_file, valid_file)in enumerate(kfold.split(data_filenames)) :
print(fold)
 training_dataset = get_dataset([data_filenames[x] for x in train_file], augment=True, shuffle=True, repeat=True, labeled=True)
 valid_dataset = get_dataset([data_filenames[x] for x in valid_file], augment=True, shuffle=False, repeat=False, labeled=False)
 K.clear_session()
with strategy.scope() :
model = create_model()
if fold == 3:
print("ok")
 optimizer = tf.keras.optimizers.Adam(lr=0.001)
 model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 callbacks = [get_lr_callback(BATCH_SIZE)]
steps_per_epoch = count_data_items([data_filenames[x] for x in train_file])/BATCH_SIZE//REPLICAS
sv = tf.keras.callbacks.ModelCheckpoint('fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='min', save_freq='epoch')
 callbacks.append(sv)
 warmup_history = model.fit(training_dataset, steps_per_epoch=steps_per_epoch, validation_data=valid_dataset, epochs=12, verbose=1, callbacks=callbacks ).history
if fold == 3:
print("whatup")
 test_dataset = get_dataset(test_filenames, augment=True, repeat=True, batch_size=BATCH_SIZE*4)
 ct_valid = count_data_items(test_filenames)
 if fold == 3:
print("hello")
 STEPS = TTA * ct_valid/32/4/REPLICAS
pred = model.predict(test_dataset,steps=STEPS,verbose=1)[:TTA*ct_valid,]
preds[:, 0] += np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1 )
train['keyword'] = train.keyword.fillna(value='None')
 train['location'] = train.location.fillna(value='None')
 test['keyword'] = test.keyword.fillna(value='None')
 test['location'] = test.location.fillna(value='None' )<load_pretrained>
preds = np.concatenate(preds )
def get_features(data, batch_size=2500):
model_class, tokenizer_class, pretrained_weights =(ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-uncased')
 tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
 model = model_class.from_pretrained(pretrained_weights)
 model.to(device)
 model = nn.DataParallel(model)
 tokenized = data["text"].apply(( lambda x: tokenizer.encode(x, add_special_tokens=True)))
 max_len = 0
for i in tokenized.values:
if len(i)> max_len:
max_len = len(i)
 padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])
 attention_mask = np.where(padded != 0, 1, 0)
 last_hidden_states=[]
no_batch = data.shape[0]//batch_size
start_index=0
end_index=1
for i in tqdm_notebook(range(1,no_batch+2)) :
if data.shape[0]>batch_size*i:
end_index=batch_size*i
else:
end_index=train.shape[0]
input_ids = torch.tensor(padded[start_index:end_index])
 batch_attention_mask = torch.tensor(attention_mask[start_index:end_index])
 input_ids = input_ids.to(device)
 batch_attention_mask = batch_attention_mask.to(device)
 with torch.no_grad() :
batch_hidden_state = model(input_ids, attention_mask=batch_attention_mask)
 print("Batch {} is completed sucessfully".format(i))
last_hidden_states.append(batch_hidden_state[0])
 start_index=batch_size*i
end_index=batch_size*i
fin_features = torch.cat(last_hidden_states,0)
 clf_features = fin_features[:,0,:].cpu().numpy()
return clf_features<feature_engineering>
preds = preds/6
gc.collect()
features = get_features(train,batch_size=2500)
 test_distil_features = get_features(test,batch_size=2500 )<define_variables>
ds = get_dataset(test_filenames, augment=False, repeat=False,
labeled=False, return_img_name=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
cat_cols = ['keyword']<concatenate>
submission = pd.DataFrame(dict(image_name=list(image_names), target=preds))
submission = submission.sort_values('image_name')
 submission = submission.set_index("image_name")
 submission.to_csv('submission.csv', index=True)
 submission.head()
train_features = combine_features(train,features,cat_cols)
 test_features = combine_features(test,test_distil_features,cat_cols )<define_variables>
<train_on_grid><EOS> 
np.zeros(( count_data_items(test_filenames),1))