Kaggle competition: Titanic - Machine Learning from Disaster
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 25
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [256, 256]
SEED = 123
LR = 0.0005
if(IMAGE_SIZE[0] == 256):
print('Using size 256')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 elif(IMAGE_SIZE[0] == 384):
print('Using size 384')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 elif(IMAGE_SIZE[0] == 512):
print('Using size 512')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 elif(IMAGE_SIZE[0] == 768):
print('Using size 768')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-768x768')
 elif(IMAGE_SIZE[0] == 1024):
print('Using size 1024')
 GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 if(IMAGE_SIZE[0] == 1024):
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')
 SUB = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<normalization>
datasets = [train,test]
for df in datasets:
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
tmp = random.uniform(0, 1)
 if 0 < tmp <= 0.1:
rot = 15.0 * tf.random.normal([1],dtype='float32')
 elif 0.1 < tmp <= 0.2:
rot = 30.0 * tf.random.normal([1],dtype='float32')
 elif 0.2 < tmp <= 0.3:
rot = 45.0 * tf.random.normal([1],dtype='float32')
 elif 0.3 < tmp <= 0.4:
rot = 60.0 * tf.random.normal([1],dtype='float32')
 elif 0.4 < tmp <= 0.5:
rot = 75.0 * tf.random.normal([1],dtype='float32')
 elif 0.5 < tmp <= 0.6:
rot = 90.0 * tf.random.normal([1],dtype='float32')
 elif 0.6 < tmp <= 0.7:
rot = 110.0 * tf.random.normal([1],dtype='float32')
 elif 0.7 < tmp <= 0.8:
rot = 130.0 * tf.random.normal([1],dtype='float32')
 elif 0.8 < tmp <= 0.9:
rot = 150.0 * tf.random.normal([1],dtype='float32')
 elif 0.9 < tmp <= 1.0:
rot = 180.0 * tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<compute_train_metric>
for df in datasets:
df['hasCabin'] = np.where(pd.isnull(df['Cabin']),0,1)
 df.loc[pd.isnull(df['Embarked']),'Embarked'] = 'None'
df.drop(['Name','Ticket','Cabin'],axis=1,inplace=True)
 train.head()
%%time
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name = 'inp2')
 efnetb6 = efn.EfficientNetB6(weights = 'imagenet', include_top = False)
 x6 = efnetb6(inp1)
 x6 = tf.keras.layers.GlobalAveragePooling2D()(x6)
 x1 = tf.keras.layers.Dense(32 )(inp2)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation('relu' )(x1)
 concat = tf.keras.layers.concatenate([x6, x1])
 concat = tf.keras.layers.Dense(256, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 concat = tf.keras.layers.Dense(64, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)],
metrics = [tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ] 
)
 print(model.summary())
 return model
def train_and_predict(SUB, folds = 5):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
tic = time.time()
print('
')
 print('-'*50)
 print(f'Training fold {fold}')
 print('-'*50)
 print('
')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 10,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.5, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 models.append(model)
 model.save_weights(f'ENet_fold_{fold}.h5')
 print('Weights saved!')
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 toc = time.time()
print('
')
 print('Time taken:', toc-tic)
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('oof.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv(f'submission.csv', index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(SUB )<compute_test_metric>
SEED = 1
np.random.seed(SEED)
 le = dict()
le['Sex'] = LabelEncoder()
le['Sex'].fit(train.Sex)
 le['Embarked'] = LabelEncoder()
le['Embarked'].fit(train.Embarked)
 le['Title'] = LabelEncoder()
le['Title'].fit(pd.concat([train.Title, test.Title], axis=0))
for df in datasets:
df['Sex'] = le['Sex'].transform(df['Sex'])
 df['Embarked'] = le['Embarked'].transform(df['Embarked'])
 df['Title'] = le['Title'].transform(df['Title'])
 train.head()
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Our out of folds roc auc score is: ', roc_auc )<import_modules>
for df in datasets:
df.loc[pd.isnull(df['Age']), 'Age'] = df['Age'].mean()
for df in datasets:
df.loc[:,'Age'] = np.round(df['Age'] )
!pip install -q efficientnet 
<set_options>
for df in datasets:
df.loc[pd.isnull(df['Fare']),'Fare'] = df['Fare'].mean()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
x_train0 = train.drop(['PassengerId','Survived'],axis=1)
 y_train0 = train['Survived']
x_test0 = test.drop(['PassengerId'],axis=1)
 sc = StandardScaler()
x_train = sc.fit_transform(x_train0)
 x_test = sc.fit_transform(x_test0)
 y_train = y_train0.values.astype('float32')
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [512, 512]
SEED = 123
LR = 0.0003
cutmix_rate = 0.3
gridmask_rate = 0
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 SUB = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<normalization>
from keras import models
from keras import layers
from keras import optimizers
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def my_ceil(a, precision=0):
return np.round(a + 0.5 * 10**(-precision), precision)
 def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot =(my_ceil(random.uniform(0, 1), 1)* 180.0)* tf.random.normal([1], dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def cutmix(image, label):
DIM = IMAGE_SIZE[0]
imgs = []; labs = []
for j in range(BATCH_SIZE):
P = tf.cast(tf.random.uniform([], 0, 1)<= cutmix_rate, tf.int32)
 k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)
 x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 b = tf.random.uniform([], 0, 1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image['inp1'][j,ya:yb,0:xa,:]
two = image['inp1'][k,ya:yb,xa:xb,:]
three = image['inp1'][j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image['inp1'][j,0:ya,:,:],middle,image['inp1'][j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, 1))
return {'inp1': image2, 'inp2': image['inp2']}, label2
def transform_gridmask(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform_gridmask(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.3
}
mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 mask = tf.cast(mask,tf.float32)
 P = tf.cast(tf.random.uniform([], 0, 1)<= PROBABILITY, tf.int32)
 if P==1:
return image*mask
else:
return image
def gridmask(img_batch, label_batch):
return {'inp1': apply_grid_mask(img_batch['inp1'],(*IMAGE_SIZE, 3)) , 'inp2': img_batch['inp2']}, label_batch
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.7, 1.3)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.map(cutmix, num_parallel_calls = AUTO)
 dataset = dataset.map(gridmask, num_parallel_calls = AUTO)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<compute_train_metric>
epochs_num = 100
batch_size = 20
input_dim = len(x_train[0] )
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name='inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name='inp2')
 efn1 = efn.EfficientNetB3(input_shape=(*IMAGE_SIZE,3), weights = 'imagenet', include_top = False)
 x1 = efn1(inp1)
 x1 = tf.keras.layers.GlobalAveragePooling2D()(x1)
 small = tf.keras.layers.AveragePooling2D(pool_size=2, strides=2 )(inp1)
 efn2 = efn.EfficientNetB4(input_shape=(256,256,3), weights='imagenet', include_top=False)
 x2 = efn2(small)
 x2 = tf.keras.layers.GlobalAveragePooling2D()(x2)
 x = tf.keras.layers.Concatenate()([x1,x2])
 xtab = tf.keras.layers.Dense(50 )(inp2)
 xtab = tf.keras.layers.BatchNormalization()(xtab)
 xtab = tf.keras.layers.Activation('relu' )(xtab)
 concat = tf.keras.layers.concatenate([x, xtab])
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer=opt, loss=[binary_focal_loss(gamma = 2.0, alpha = 0.80)], metrics=[tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ])
 return model
def train_and_predict(SUB, folds = 3):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset, steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS, callbacks = [early_stopping, cb_lr_schedule], validation_data = val_dataset, verbose = 2)
 models.append(model)
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('EfficientNetB4_5_ensemble.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv('sub_EfficientNetB4_5_ensemble.csv', index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(SUB )<compute_test_metric>
def get_model(input_dim):
model = models.Sequential()
model.add(layers.Dense(units = 7, kernel_initializer = 'lecun_uniform', activation = 'relu', input_dim = input_dim))
model.add(layers.Dense(units = 5, kernel_initializer = 'lecun_uniform', activation = 'relu'))
model.add(layers.Dense(units = 1, kernel_initializer = 'lecun_uniform', activation = 'sigmoid'))
model.compile(optimizer=optimizers.Adam(learning_rate=0.01),
loss='binary_crossentropy', metrics=['accuracy'])
 return model
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Our out of folds roc auc score is: ', roc_auc )<set_options>
model = get_model(input_dim)
 history = model.fit(x_train, y_train, epochs=epochs_num, batch_size=batch_size, verbose=1)
warnings.filterwarnings('ignore')
<load_from_csv>
predict = model.predict(x_test )
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 SEED = 22
seed_everything(SEED)
 def read_data() :
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 train1 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB0_256.csv')
 train2 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB0_512EX.csv')
 train3 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB1_256.csv')
 train4 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB1_384.csv')
 train5 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB1_512.csv')
 train6 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB1_512EX.csv')
 train7 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB2_256.csv')
 train8 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB2_384.csv')
 train9 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB2_512.csv')
 train10 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB2_512EX.csv')
 train11 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB3_256.csv')
 train12 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB3_384.csv')
 train13 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB3_512.csv')
 train14 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB4_256.csv')
 train15 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB4_384.csv')
 train16 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB4_512.csv')
 train17 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB5_256.csv')
 train18 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB5_384.csv')
 train19 = pd.read_csv('.. /input/melanoma-subs/EfficientNetB6_384.csv')
 test1 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB0_256.csv')
 test2 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB0_512EX.csv')
 test3 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB1_256.csv')
 test4 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB1_384.csv')
 test5 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB1_512.csv')
 test6 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB1_512EX.csv')
 test7 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB2_256.csv')
 test8 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB2_384.csv')
 test9 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB2_512.csv')
 test10 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB2_512EX.csv')
 test11 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB3_256.csv')
 test12 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB3_384.csv')
 test13 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB3_512.csv')
 test14 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB4_256.csv')
 test15 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB4_384.csv')
 test16 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB4_512.csv')
 test17 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB5_256.csv')
 test18 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB5_384.csv')
 test19 = pd.read_csv('.. /input/melanoma-subs/sub_EfficientNetB6_384.csv')
 def print_roc_auc(df, model):
roc_auc = metrics.roc_auc_score(df['target'], df['predictions'])
 print(f'Our model {model} out of folds roc auc score is {roc_auc}')
 print('-'*50)
 print('
')
 print_roc_auc(train1, 1)
 print_roc_auc(train2, 2)
 print_roc_auc(train3, 3)
 print_roc_auc(train4, 4)
 print_roc_auc(train5, 5)
 print_roc_auc(train6, 6)
 print_roc_auc(train7, 7)
 print_roc_auc(train8, 8)
 print_roc_auc(train9, 9)
 print_roc_auc(train10, 10)
 print_roc_auc(train11, 11)
 print_roc_auc(train12, 12)
 print_roc_auc(train13, 13)
 print_roc_auc(train14, 14)
 print_roc_auc(train15, 15)
 print_roc_auc(train16, 16)
 print_roc_auc(train17, 17)
 print_roc_auc(train18, 18)
 print_roc_auc(train19, 19)
 def fix_predictions(train, test, model):
test.columns = ['image_name', 'predictions_{}'.format(model)]
train = train[['image_name', 'predictions']]
train.columns = ['image_name', 'predictions_{}'.format(model)]
return train, test
train1, test1 = fix_predictions(train1, test1, 1)
 train2, test2 = fix_predictions(train2, test2, 2)
 train3, test3 = fix_predictions(train3, test3, 3)
 train4, test4 = fix_predictions(train4, test4, 4)
 train5, test5 = fix_predictions(train5, test5, 5)
 train6, test6 = fix_predictions(train6, test6, 6)
 train7, test7 = fix_predictions(train7, test7, 7)
 train8, test8 = fix_predictions(train8, test8, 8)
 train9, test9 = fix_predictions(train9, test9, 9)
 train10, test10 = fix_predictions(train10, test10, 10)
 train11, test11 = fix_predictions(train11, test11, 11)
 train12, test12 = fix_predictions(train12, test12, 12)
 train13, test13 = fix_predictions(train13, test13, 13)
 train14, test14 = fix_predictions(train14, test14, 14)
 train15, test15 = fix_predictions(train15, test15, 15)
 train16, test16 = fix_predictions(train16, test16, 16)
 train17, test17 = fix_predictions(train17, test17, 17)
 train18, test18 = fix_predictions(train18, test18, 18)
 train19, test19 = fix_predictions(train19, test19, 19)
 train = train.merge(train1, on = 'image_name' ).merge(train2, on = 'image_name' ).merge(train3, on = 'image_name' ).merge(train4, on = 'image_name' ).merge(train5, on = 'image_name' ).merge(train6, on = 'image_name' ).merge(train7, on = 'image_name' ).merge(train8, on = 'image_name' ).merge(train9, on = 'image_name' ).merge(train10, on = 'image_name' ).merge(train11, on = 'image_name' ).merge(train12, on = 'image_name' ).merge(train13, on = 'image_name' ).merge(train14, on = 'image_name' ).merge(train15, on = 'image_name' ).merge(train16, on = 'image_name' ).merge(train17, on = 'image_name' ).merge(train18, on = 'image_name' ).merge(train19, on = 'image_name')
 test = test.merge(test1, on = 'image_name' ).merge(test2, on = 'image_name' ).merge(test3, on = 'image_name' ).merge(test4, on = 'image_name' ).merge(test5, on = 'image_name' ).merge(test6, on = 'image_name' ).merge(test7, on = 'image_name' ).merge(test8, on = 'image_name' ).merge(test9, on = 'image_name' ).merge(test10, on = 'image_name' ).merge(test11, on = 'image_name' ).merge(test12, on = 'image_name' ).merge(test13, on = 'image_name' ).merge(test14, on = 'image_name' ).merge(test15, on = 'image_name' ).merge(test16, on = 'image_name' ).merge(test17, on = 'image_name' ).merge(test18, on = 'image_name' ).merge(test19, on = 'image_name')
 return train, test, sub
def feature_engineering(train, test):
trn_images = train['image_name'].values
trn_sizes = np.zeros(( trn_images.shape[0], 2))
for i, img_path in enumerate(tqdm(trn_images)) :
img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/', f'{img_path}.jpg'))
trn_sizes[i] = np.array([img.size[0], img.size[1]])
 test_images = test['image_name'].values
test_sizes = np.zeros(( test_images.shape[0],2))
for i, img_path in enumerate(tqdm(test_images)) :
img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/test/', f'{img_path}.jpg'))
test_sizes[i] = np.array([img.size[0],img.size[1]])
 train['w'] = trn_sizes[:,0]
train['h'] = trn_sizes[:,1]
test['w'] = test_sizes[:,0]
test['h'] = test_sizes[:,1]
return train, test
def encode_categorical(train, test):
for col in ['sex', 'anatom_site_general_challenge']:
encoder = preprocessing.LabelEncoder()
train[col].fillna('unknown', inplace = True)
 test[col].fillna('unknown', inplace = True)
 train[col] = encoder.fit_transform(train[col])
 test[col] = encoder.transform(test[col])
 train['patient_id'].fillna('unknown', inplace = True)
 return train, test
def train_and_evaluate_lgbm(train, test, params, verbose_eval, folds = 5):
features = [col for col in train.columns if col not in ['image_name', 'patient_id', 'diagnosis', 'benign_malignant', 'target', 'source']]
if verbose_eval != False:
print('Training with features: ', features)
 kf = GroupKFold(n_splits = folds)
 target = 'target'
oof_pred = np.zeros(len(train))
y_pred = np.zeros(len(test))
for fold,(tr_ind, val_ind)in enumerate(kf.split(train, groups = train['patient_id'])) :
if verbose_eval != False:
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}"')
 x_train, x_val = train[features].iloc[tr_ind], train[features].iloc[val_ind]
y_train, y_val = train[target][tr_ind], train[target][val_ind]
train_set = lgb.Dataset(x_train, y_train)
 val_set = lgb.Dataset(x_val, y_val)
 model = lgb.train(params, train_set, num_boost_round = 10000, early_stopping_rounds = 50,
valid_sets = [train_set, val_set], verbose_eval = verbose_eval)
 oof_pred[val_ind] = model.predict(x_val)
 y_pred += model.predict(test[features])/ kf.n_splits
rauc = metrics.roc_auc_score(train['target'], oof_pred)
 if verbose_eval != False:
print(f'Our oof roc auc score for our lgbm model is {rauc}')
 gc.collect()
return rauc, y_pred
train, test, sub = read_data()
train, test = feature_engineering(train, test)
 train, test = encode_categorical(train, test )<choose_model_class>
my_submission = pd.DataFrame({
	'PassengerId': test.PassengerId,
	'Survived': pd.Series(predict.reshape(( 1,-1)) [0] ).round().astype(int)
 })
 my_submission.head()
<install_modules><EOS> 
my_submission.to_csv('submission.csv', index=False )