Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<string_transform>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
reverse_word_index = dict([(value, key)for(key, value)in word_index.items() ])
 def decode_sentence(text):
return ' '.join([reverse_word_index.get(i, '?')for i in text])
 print(decode_sentence(padded[0]))<define_variables>
from sklearn import metrics
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras.layers as L
from kaggle_datasets import KaggleDatasets
from xgboost import XGBClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import KFold
import random
from functools import reduce
import scipy
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \
-O /tmp/glove.6B.100d.txt
embeddings_index = {};
vocab_size=len(word_index)
 embedding_dim = 100
with open('/tmp/glove.6B.100d.txt')as f:
for line in f:
values = line.split() ;
word = values[0];
coefs = np.asarray(values[1:], dtype='float32');
embeddings_index[word] = coefs;
embeddings_matrix = np.zeros(( vocab_size+1, embedding_dim)) ;
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word);
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector;<choose_model_class>
def get_df(file,name):
try:
df = pd.read_csv(file.format(name))
except FileNotFoundError:
df = pd.read_csv(file.format(''))
return df
def read_oof(data_ext,name=''):
return [get_df('.. /input/'+file+'/{}oof.csv',name ).set_index('image_name')for file in data_ext]
def read_pred(attach,mean,name=''):
df = get_df('.. /input/'+attach + '/{}preds_all.csv',name)
 img_col = np.where(df.dtypes=='object')[0][0]
df.columns.values[img_col]='0'
df = pd.DataFrame(dict(image_name=df['0'], target=mean(df.drop(['0'],axis=1),axis=1)))
 return df.set_index('image_name')
 def read_preds(data_ext,mean,name=''):
return [read_pred(file,mean,name=name)for file in data_ext]
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D() ,
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy',tf.keras.metrics.AUC() ])
 model.summary()<train_model>
data_ext = ['meta-128-assemble',
'meta-192',
'bs-384-combine',
'sls-assemble',
'meta-512-assemble',
'meta-768-ls-0-05-assemble',
'meta-1026-ls-0-05-assemble',
'meta-data-xgb']
mn_func = scipy.stats.gmean
sub_list = read_preds(data_ext,mn_func)
 oof_list = read_oof(data_ext,'oof' )
num_epochs = 3
history = model.fit(padded, y_train, epochs=num_epochs, validation_data=(testing_padded, y_val))<prepare_output>
def collect(df_list):
return pd.concat(df_list,axis=1)
 sub_df = collect(sub_list)
 oof_df = collect(oof_list)
 oof_df = oof_df.dropna()
ids = oof_df.index
oof = np.array(oof_df['pred'])
 labels = np.array(oof_df['target'])[:,0]
print('There are {} values overlapping'.format(len(oof_df)))
 idsO = sub_df.index
data_sub = np.array(sub_df )
model_loss = pd.DataFrame(model.history.history)
 model_loss.head()<string_transform>
DIM = 256; EFFN = 0; BATCH_SIZE = 128
PATH_TO_EMBEDDINGS = '.. /input/embeddingsmelanoma/'
embed = np.load(PATH_TO_EMBEDDINGS+'embed_train_%i_%i.npy'%(DIM,EFFN))
names = np.load(PATH_TO_EMBEDDINGS+'names_train.npy')
 embed_test = np.load(PATH_TO_EMBEDDINGS+'embed_test_%i_%i.npy'%(DIM,EFFN))
names_test = np.load(PATH_TO_EMBEDDINGS+'names_test.npy' )
testing_sequences2 = tokenizer.texts_to_sequences(test_df.text)
 testing_padded2 = pad_sequences(testing_sequences2, maxlen=max_length, padding=padding_type, truncating=trunc_type )<predict_on_test>
test = pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv' ).set_index('image_name',drop=True)
 test = test.loc[names_test]
test = pd.concat(( test,sub_df),axis=1 ).reset_index()
print('Test csv shape',test.shape)
 train = pd.read_csv('.. /input/melanoma-%ix%i/train.csv'%(DIM,DIM)).set_index('image_name',drop=True)
 train = train.loc[names]
train = pd.concat(( train,oof_df['pred']),axis=1 ).reset_index()
train.target = train.target.astype('float32')
 print('Train csv shape',train.shape)
 print('Displaying train.csv below...')
 train.head()
probabilities = model.predict(testing_padded2 )<data_type_conversions>
cat_enc = OneHotEncoder(drop='first')
 num_enc = StandardScaler()
embed_enc = MinMaxScaler()
numeric_features = ['age_approx']
cat_features = ['sex','anatom_site_general_challenge']
cats = cat_enc.fit_transform(train[cat_features].fillna('0')).toarray()
nums = num_enc.fit_transform(train[numeric_features].fillna(0))
embed_ENC = embed_enc.fit_transform(embed)
 Xtrain = np.concatenate(( cats,nums,embed_ENC),axis=1)
 Xpreds = np.array(train['pred'])
 cats = cat_enc.transform(test[cat_features].fillna('0')).toarray()
nums = num_enc.transform(test[numeric_features].fillna(0))
embed_encT = embed_enc.transform(embed_test)
 Xtest = np.concatenate(( cats,nums,embed_encT),axis=1)
 Xtargets = np.array(test['target'] )
predictions =(probabilities > 0.5 ).astype(int)
 predictions = np.ndarray.flatten(predictions)
 pd.value_counts(predictions )<save_to_csv>
weights = {0:1, 1:15}
FOLDS=5
SEED=42
DISPLAY_PLOT = 1
REPLICAS=1
EPOCHS=100
TTA=11
batch_size = 64
VERBOSE=0
original_test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df = pd.DataFrame({'text' : original_test_df['text'],'cleaned_text' : test_df['text'], 'prediction' : predictions,'probabilities' : np.ndarray.flatten(probabilities)})
 df.to_csv("test_df.csv", index=False )<save_to_csv>
tf.random.set_seed(5);
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = predictions
sample_submission.to_csv("submission.csv", index=False )<import_modules>
def data_augment(data,mean=0.0,std1=embed_std/10,std2=0.03):
gauss = np.random.normal(loc=mean, scale=std1, size=Xtrain.shape[1])
 new_data = data[0] + gauss
gauss = np.random.normal(loc=mean, scale=std2, size=Xpreds.shape[1])
 new_preds = data[1] + gauss
return(new_data,new_preds )
import tensorflow_hub as hub
import lightgbm as lgb
from lightgbm import LGBMClassifier<load_pretrained>
def get_dataset(X,subs,y,augment=True,repeat=True,batch=batch_size):
ds = tf.data.Dataset.from_tensor_slices(((X,subs), y))
if repeat:
ds = ds.repeat()
if augment:
ds = ds.map(lambda elem,label:(data_augment(elem),label))
ds = ds.batch(batch)
 return ds
module_url = "https://tfhub.dev/google/nnlm-en-dim128/2"
embed = hub.KerasLayer(module_url)
 embeddings = embed(["A long sentence.", "single-word",
"http://example.com"])
 print(embeddings.shape )<load_pretrained>
dim = Xtrain.shape[1]
meta_dim = Xpreds.shape[1]
def build_model(ls=0.05):
inp = tf.keras.layers.Input(shape=(dim,))
x = L.Dropout(0.2 )(inp)
 x = L.Dense(int(1024), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(int(512), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(int(256), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(int(128), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(64, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(32, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(16, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(12, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 meta_inp = tf.keras.layers.Input(shape=(meta_dim,))
xm = L.concatenate(( x,meta_inp))
xm = L.Dense(16, activation='relu' )(xm)
 xm = L.Dropout(0.2 )(xm)
 xm = L.Dense(12, activation='relu' )(xm)
 xm = L.Dropout(0.2 )(xm)
 xm = L.Dense(8, activation='relu' )(xm)
 xm = L.Dropout(0.1 )(xm)
 xm = L.Dense(1, activation='sigmoid' )(xm)
 model = tf.keras.Model(inputs=(inp,meta_inp), outputs=xm)
 opt = tf.keras.optimizers.Adam(learning_rate=0.00000125* REPLICAS * batch_size)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=ls)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/3" )<categorify>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 50
lr_sus_ep = 0
lr_decay = 0.99
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
<init_hyperparams><EOS> 
ids = train['index']
labels = train.target.astype('int32')
 idsO = test['index']
data_sub = Xtest
strategy = tf.distribute.get_strategy()
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( test.shape[0],1))
preds_all = np.zeros(( test.shape[0],FOLDS))
for fold,(idxT2,idxV2)in enumerate(skf.split(np.arange(15))):
idxT = train.loc[train.tfrecord.isin(idxT2)].index.values
idxV = train.loc[train.tfrecord.isin(idxV2)].index.values
X = Xtrain[idxT]; y = labels[idxT]; XX=Xpreds[idxT]
X_val = Xtrain[idxV]; y_val = labels[idxV]; XX_val=Xpreds[idxV]
tf.keras.backend.clear_session()
with strategy.scope() :
model=build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 history = model.fit(get_dataset(X,XX,y),epochs=EPOCHS,steps_per_epoch=X.shape[0]/batch_size//REPLICAS,
verbose=VERBOSE,class_weight=weights,
callbacks=[sv],
validation_data=get_dataset(X_val,XX_val, y_val,augment=False,repeat=False))
model.load_weights('fold-%i.h5'%fold)
 STEPS = TTA*X_val.shape[0]/(batch_size-1)/REPLICAS
pred = model.predict(get_dataset(X_val,XX_val,y_val), steps=STEPS)[:TTA*X_val.shape[0]]
oof_pred.append(np.mean(pred.reshape(( X_val.shape[0],TTA),order='F'),axis=1))
oof_tar.append(y_val)
 oof_names.append(ids[idxV])
 oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 STEPS = TTA*Xtest.shape[0]/(batch_size-1)/REPLICAS
psub = model.predict(get_dataset(Xtest,Xtargets,np.zeros(len(Xtest))), steps=STEPS)[:TTA*Xtest.shape[0]]
pstore = np.mean(psub.reshape(( len(Xtest),TTA),order='F'),axis=1)
 preds[:,0] += pstore*1/FOLDS
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.legend(loc=3)
 plt.show()
oof_c = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof_c)
 print('Overall OOF AUC with TTA = %.5f'%auc)
 dd_oof = pd.DataFrame(dict(image_name = names, target=true, pred = oof_c, fold=folds))
submission = pd.DataFrame(dict(image_name=idsO, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()