Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_train_metric>
SEED = 7
print("Setup complete." )
def sigmoid_focal_cross_entropy_with_logits(
 labels, logits, alpha=0.25, gamma=2.0):
if gamma and gamma < 0:
raise ValueError("Value of gamma should be greater than or equal to zero")
 logits = tf.convert_to_tensor(logits)
 labels = tf.convert_to_tensor(labels, dtype=logits.dtype)
 ce = tf.nn.sigmoid_cross_entropy_with_logits(
 labels=labels, logits=logits)
 pred_prob = tf.math.sigmoid(logits)
 p_t =(labels * pred_prob)+(( 1 - labels)*(1 - pred_prob))
alpha_factor = 1.0
modulating_factor = 1.0
if alpha:
alpha = tf.convert_to_tensor(alpha, dtype=tf.float32)
 alpha_factor = labels * alpha +(1 - labels)*(1 - alpha)
 if gamma:
gamma = tf.convert_to_tensor(gamma, dtype=tf.float32)
 modulating_factor = tf.pow(( 1.0 - p_t), gamma)
 return tf.math.reduce_sum(
 alpha_factor * modulating_factor * ce, axis=-1)
 class NeuralNet(tf.keras.Model):
def __init__(self, engine, input_shape, pretrained_weights):
super(NeuralNet, self ).__init__()
self.engine = engine(
 include_top=False,
input_shape=input_shape,
weights=pretrained_weights)
 self.pool = tf.keras.layers.GlobalAveragePooling2D()
self.concat = tf.keras.layers.Concatenate()
self.sequential_meta = tf.keras.Sequential([
tf.keras.layers.Dense(64),
tf.keras.layers.ReLU() ,
])
 self.sequential_merged = tf.keras.Sequential([
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(1, dtype='float32')
 ])
 def call(self, inputs, **kwargs):
if isinstance(inputs, dict):
images = inputs['image']
site = inputs['anatom_site_general_challenge']
sex = inputs['sex']
age = inputs['age_approx']
else:
images = inputs[0]
site = inputs[1]
sex = inputs[2]
age = inputs[3]
x1 = self.engine(images)
 x1 = self.pool(x1)
 x2 = tf.concat([site, sex, age], axis=-1)
 x2 = self.sequential_meta(x2)
 x3 = self.concat([x1, x2])
 x3 = self.sequential_merged(x3)
 return x3
class DistributedModel:
def __init__(self,
engine,
input_shape=(384, 384, 3),
pretrained_weights=None,
finetuned_weights=None,
batch_size=8,
optimizer=None,
strategy=None,
mixed_precision=False,
label_smoothing=0.0,
tta=1,
focal_loss=True,
save_best=None):
self.keras_model = NeuralNet(
 engine=engine,
input_shape=input_shape,
pretrained_weights=pretrained_weights)
 self.keras_model.build(
 [[None, *input_shape], [None, 7], [None, 1], [None, 1]])
 if finetuned_weights:
self.keras_model.load_weights(finetuned_weights)
 self._initial_weights = self.keras_model.get_weights()
self.global_batch_size = batch_size
self.mixed_precision = mixed_precision
self.optimizer = optimizer
self.strategy = strategy
self.label_smoothing = label_smoothing
self.tta = tta
self.focal_loss = focal_loss
self.save_best = save_best
self.auc_metric = tf.keras.metrics.AUC()
self.loss_metric = tf.keras.metrics.Mean()
if self.optimizer and self.mixed_precision:
self.optimizer = \
tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, loss_scale='dynamic')
 if self.strategy:
self.global_batch_size *= self.strategy.num_replicas_in_sync
if save_best and not(os.path.isdir(save_best)) :
os.makedirs(save_best)
 def reset_weights(self):
self.keras_model.set_weights(self._initial_weights)
 def _compute_loss(self, labels, logits):
if self.focal_loss:
per_example_loss = sigmoid_focal_cross_entropy_with_logits(
 labels=labels, logits=logits, alpha=0.8, gamma=2.0)
 else:
per_example_loss = tf.nn.sigmoid_cross_entropy_with_logits(
 labels=labels, logits=logits)
 return tf.nn.compute_average_loss(
 per_example_loss, global_batch_size=self.global_batch_size)
 @tf.function
def _distributed_train_step(self, dist_inputs):
def train_step(inputs):
if self.label_smoothing:
target =(
 inputs['target'] *(1 - self.label_smoothing)
 + 0.5 * self.label_smoothing 
)
 else:
target = inputs['target']
with tf.GradientTape() as tape:
logits = self.keras_model(inputs, training=True)
 loss = self._compute_loss(target, logits)
 self.loss_metric.update_state(loss)
 self.auc_metric.update_state(
 tf.math.round(target), tf.math.sigmoid(logits))
if self.mixed_precision:
scaled_loss = self.optimizer.get_scaled_loss(loss)
 if self.mixed_precision:
scaled_gradients = tape.gradient(
 scaled_loss, self.keras_model.trainable_variables)
 gradients = self.optimizer.get_unscaled_gradients(scaled_gradients)
 else:
gradients = tape.gradient(loss, self.keras_model.trainable_variables)
 self.optimizer.apply_gradients(
 zip(gradients, self.keras_model.trainable_variables))
return loss
per_replica_loss = self.strategy.run(train_step, args=(dist_inputs,))
return self.strategy.reduce(
 tf.distribute.ReduceOp.SUM, per_replica_loss, axis=None)
 @tf.function
def _distributed_predict_step(self, dist_inputs):
def predict_step(inputs):
logits = self.keras_model(inputs, training=False)
 return tf.math.sigmoid(logits), inputs['image_name'], inputs['target']
preds, image_names, trues = self.strategy.run(predict_step, args=(dist_inputs,))
if tf.is_tensor(preds):
return [preds], [image_names], [trues]
else:
return preds.values, image_names.values, trues.values
def fit(self, ds):
ds = self.strategy.experimental_distribute_dataset(ds)
 ds = tqdm.tqdm(ds)
 for i, inputs in enumerate(ds):
loss = self._distributed_train_step(inputs)
 ds.set_description(
 "valid AUC {:.4f} : Loss/AUC [{:.4f}, {:.4f}]".format(
 self.auc_score,
self.loss_metric.result().numpy() ,
self.auc_metric.result().numpy() 
)
)
 self.loss_metric.reset_states()
self.auc_metric.reset_states()
def predict(self, ds):
ds = self.strategy.experimental_distribute_dataset(ds.repeat(self.tta))
ds = tqdm.tqdm(ds)
 preds_accum = np.zeros([0, 1], dtype=np.float32)
 names_accum = np.zeros([0, 1], dtype=str)
 trues_accum = np.zeros([0, 1], dtype=np.float32)
 for inputs in ds:
preds, names, trues = self._distributed_predict_step(inputs)
 for pred, name, true in zip(preds, names, trues):
preds_accum = np.concatenate([preds_accum, pred.numpy() ], axis=0)
 names_accum = np.concatenate([names_accum, name.numpy() ], axis=0)
 trues_accum = np.concatenate([trues_accum, true.numpy() ], axis=0)
 preds_accum = preds_accum.reshape(( self.tta, -1)).mean(axis=0)
 names_accum = names_accum.reshape(( self.tta, -1)) [0]
trues_accum = trues_accum.reshape(( self.tta, -1)).mean(axis=0 ).round()
return preds_accum, names_accum, trues_accum
def fit_and_predict(self, fold, epochs, train_ds, valid_ds, test_ds):
self.auc_score = 0.
 self.best_score = 0.
 for epoch in range(epochs):
self.fit(train_ds)
 if epoch >= 9:
valid_preds, valid_names, valid_trues = self.predict(valid_ds)
 self.auc_score = metrics.roc_auc_score(valid_trues, valid_preds)
 if self.auc_score > self.best_score:
self.best_score = self.auc_score
best_valid_preds = valid_preds.copy()
if self.save_best:
self.keras_model.save_weights(
 self.save_best+f'{self.keras_model.layers[0].name}-{fold}-{epoch}.h5')
 test_preds, test_names, _ = self.predict(test_ds)
 return best_valid_preds, valid_names, test_preds, test_names<choose_model_class>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 datasets = [train, test]
train
def get_optimizer(steps_per_epoch, lr_max, lr_min,
decay_epochs, warmup_epochs, power=1):
if decay_epochs > 0:
learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(
 initial_learning_rate=lr_max,
decay_steps=steps_per_epoch*decay_epochs,
end_learning_rate=lr_min,
power=power, 
)
 else:
learning_rate_fn = lr_max
if warmup_epochs > 0:
learning_rate_fn = WarmUp(
 lr_start = lr_min,
lr_end = lr_max,
lr_fn = learning_rate_fn,
warmup_steps=steps_per_epoch*warmup_epochs,
power=power, 
)
 return tf.keras.optimizers.Adam(learning_rate_fn)
 class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, lr_start, lr_end, lr_fn, warmup_steps, power=1):
super().__init__()
self.lr_start = lr_start
self.lr_end = lr_end
self.lr_fn = lr_fn
self.warmup_steps = warmup_steps
self.power = power
def __call__(self, step):
global_step_float = tf.cast(step, tf.float32)
 warmup_steps_float = tf.cast(self.warmup_steps, tf.float32)
 warmup_percent_done = global_step_float / warmup_steps_float
warmup_learning_rate = tf.add(tf.multiply(
 self.lr_start-self.lr_end,
tf.math.pow(1-warmup_percent_done, self.power)) , self.lr_end)
 return tf.cond(
 global_step_float < warmup_steps_float,
lambda: warmup_learning_rate,
lambda: self.lr_fn(step), 
)
 def get_config(self):
return {
"lr_start": self.lr_start,
"lr_end": self.lr_end,
"lr_fn": self.lr_fn,
"warmup_steps": self.warmup_steps,
"power": self.power,
}<init_hyperparams>
for ds in datasets:
def rand_ages() :
np.random.seed(SEED)
 return np.random.randint(low=ds['Age'].mean() - ds['Age'].std() ,
high=ds['Age'].mean() + ds['Age'].std() ,
size=ds['Age'].isnull().sum())
 ds.loc[ds['Age'].isnull() , 'Age'] = rand_ages()
ds['Age'] = pd.cut(
 ds['Age'], bins=[-np.inf, 14, 24, 64, np.inf], labels=range(4))
ds.loc[:, 'Age'] = ds['Age'].astype(int)
 ax = sns.countplot(data=train, x='Age', hue='Survived')
 ax.set_xlabel('Age category')
 plt.show()
config = {
'lr_max': 3e-4,
'lr_min': 3e-5,
'lr_decay_epochs': 14,
'lr_warmup_epochs': 1,
'lr_decay_power': 1,
'n_epochs': 12,
'label_smoothing': 0.05,
'focal_loss': False,
'tta': 5,
'save_best': None,
'pretrained_weights': 'imagenet',
'finetuned_weights': None,
}
fold_config = {
0: {
'engine': EfficientNetB0,
'input_path': 'melanoma-512x512/',
'input_shape':(512, 512, 3),
'batch_size': 32,
},
1: {
'engine': EfficientNetB0,
'input_path': 'melanoma-384x384/',
'input_shape':(384, 384, 3),
'batch_size': 32,
},
2: {
'engine': EfficientNetB0,
'input_path': 'melanoma-256x256/',
'input_shape':(256, 256, 3),
'batch_size': 32,
},
3: {
'engine': EfficientNetB1,
'input_path': 'melanoma-384x384/',
'input_shape':(384, 384, 3),
'batch_size': 32,
},
4: {
'engine': EfficientNetB2,
'input_path': 'melanoma-384x384/',
'input_shape':(384, 384, 3),
'batch_size': 24,
},
}
splits = model_selection.KFold(
 len(fold_config), shuffle=True, random_state=42 ).split(X=range(15))
with strategy.scope() :
valid_preds_accum, test_preds_accum, test_names_accum = list() , list() , list()
for fold,(train_idx, valid_idx)in enumerate(splits):
optimizer = get_optimizer(
 steps_per_epoch=33126//fold_config[fold]['batch_size'],
lr_max=config['lr_max'],
lr_min=config['lr_min'],
decay_epochs=config['lr_decay_epochs'],
warmup_epochs=config['lr_warmup_epochs'],
power=config['lr_decay_power'] 
)
 dist_model = DistributedModel(
 engine=fold_config[fold]['engine'],
input_shape=fold_config[fold]['input_shape'],
pretrained_weights=config['pretrained_weights'],
finetuned_weights=config['finetuned_weights'],
batch_size=fold_config[fold]['batch_size'],
optimizer=optimizer,
strategy=strategy,
mixed_precision=mixed_precision,
label_smoothing=config['label_smoothing'],
tta=config['tta'],
focal_loss=config['focal_loss'],
save_best=config['save_best'])
 tfrec_paths = np.asarray(
 glob.glob(input_path+fold_config[fold]['input_path']+'train[0-9]*'))
test_paths = glob.glob(
 input_path+fold_config[fold]['input_path']+'test[0-9]*')
 train_paths = tfrec_paths[train_idx]
valid_paths = tfrec_paths[valid_idx]
train_ds = get_dataset(
 train_paths, fold_config[fold]['batch_size'], augment=True, shuffle=True)
 valid_ds = get_dataset(
 valid_paths, fold_config[fold]['batch_size'], augment=True)
 test_ds = get_dataset(
 test_paths, fold_config[fold]['batch_size'], augment=True)
 valid_preds, _, test_preds, test_names = dist_model.fit_and_predict(
 fold=fold,
epochs=config['n_epochs'],
train_ds=train_ds,
valid_ds=valid_ds,
test_ds=test_ds, 
)
 valid_preds_accum.append(valid_preds)
 test_preds_accum.append(test_preds)
 test_names_accum.append(test_names)
<save_to_csv>
def encode_freq_sorted(feature):
sorted_indices = feature.value_counts().index
sorted_dict = dict(zip(sorted_indices, range(len(sorted_indices))))
return feature.map(sorted_dict ).astype(int)
 for ds in datasets:
ds['Sex'] = encode_freq_sorted(ds['Sex'])
 ds['Embarked'] = encode_freq_sorted(ds['Embarked'])
 ds['Title'] = encode_freq_sorted(ds['Title'])
 train.head()
final_preds = np.average(test_preds_accum, axis=0, weights=[1,1,1,1,1])
 final_preds_map = dict(zip(test_names_accum[0].astype('U13'), final_preds))
submission_data['target'] = submission_data.image_name.map(final_preds_map)
 submission_data.to_csv('submission.csv', index=False )<import_modules>
drop_features = ['Name', 'SibSp', 'Parch', 'Ticket', 'Cabin']
import pandas as pd
import numpy as np<import_modules>
drop_features.extend(['Pclass'])
 train = train.drop(columns=drop_features)
 test = test.drop(columns=drop_features)
 X = train.drop(columns=['PassengerId', 'Survived'])
 y = train['Survived']
X.head()
import pandas as pd
import numpy as np<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(X, y, train_size=0.75,
random_state=SEED)
 X_test = test.drop(columns=['PassengerId'] )
sub1 = pd.read_csv('.. /input/melanoma-dif-sub/pl_0.936.csv')
 sub2 = pd.read_csv('.. /input/melanoma-dif-sub/pl_0.940.csv')
 sub3 = pd.read_csv('.. /input/melanoma-dif-sub/sub_EfficientNetB2_384.csv')
 sub4 = pd.read_csv('.. /input/melanoma-dif-sub/sub_EfficientNetB3_384.csv')
 sub5 = pd.read_csv('.. /input/melanoma-dif-sub/sub_EfficientNetB3_384_v2.csv')
 def rank_data(sub):
sub['target'] = sub['target'].rank() / sub['target'].rank().max()
return sub
sub1 = rank_data(sub1)
 sub2 = rank_data(sub2)
 sub3 = rank_data(sub3)
 sub4 = rank_data(sub4)
 sub5 = rank_data(sub5)
 sub1.columns = ['image_name', 'target1']
sub2.columns = ['image_name', 'target2']
sub3.columns = ['image_name', 'target3']
sub4.columns = ['image_name', 'target4']
sub5.columns = ['image_name', 'target5']
f_sub = sub1.merge(sub2, on = 'image_name' ).merge(sub3, on = 'image_name' ).merge(sub4, on = 'image_name' ).merge(sub5, on = 'image_name')
 f_sub['target'] = f_sub['target1'] * 0.3 + f_sub['target2'] * 0.3 + f_sub['target3'] * 0.05 + f_sub['target4'] * 0.3 + f_sub['target5'] * 0.05
f_sub = f_sub[['image_name', 'target']]
f_sub.to_csv('blend_sub.csv', index = False )<install_modules>
cross_valid = StratifiedKFold(n_splits=3, shuffle=True, random_state=SEED)
 def random_search(X, y, estimator, params, score="accuracy", cv=cross_valid,
n_iter=100, random_state=SEED, n_jobs=-1):

 print("
classifier = RandomizedSearchCV(estimator=estimator, param_distributions=params,
scoring=score, cv=cv, n_iter=n_iter, n_jobs=n_jobs,
random_state=random_state)
 classifier.fit(X, y)
 print("Best parameters by random search:
", classifier.best_params_)
 return classifier
!pip install -q efficientnet<import_modules>
random_forest = RandomForestClassifier(random_state=SEED)
 random_forest.get_params()
import os
import re
import numpy as np
import pandas as pd
import random
import math
import matplotlib.pyplot as plt
from sklearn import metrics
from sklearn.model_selection import KFold, StratifiedKFold
import tensorflow as tf
from kaggle_datasets import KaggleDatasets
import efficientnet.tfkeras as efn
import dill
from tensorflow.keras import backend as K
import tensorflow_addons as tfa<choose_model_class>
params = {
'bootstrap': [True, False],
'max_depth': [int(x)for x in np.linspace(10, 110, num = 11)],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [int(x)for x in np.linspace(200, 2000, num = 10)]
}
random_forest_tuned = random_search(
 X_train, y_train, estimator=random_forest, params=params )
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )<load_from_csv>
y_pred = random_forest_tuned.predict(X_val)
 accuracy_random_forest = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_random_forest )
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 EPOCHS = 30
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [384, 384]
SEED = 333
LR = 1e-5
cutmix_rate = 0.30
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 submission = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<normalization>
svc = SVC(probability=True, random_state=SEED)
 svc.get_params()
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
rot = 15.* tf.random.normal([1],dtype='float32')
 else:
rot = 180.* tf.random.normal([1],dtype='float32')
 if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
shr = 5.* tf.random.normal([1],dtype='float32')
 else:
shr = 2.* tf.random.normal([1],dtype='float32')
 if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 else:
h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.
 if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 else:
w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.
 if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
h_shift = 16.* tf.random.normal([1],dtype='float32')
 else:
h_shift = 8.* tf.random.normal([1],dtype='float32')
 if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
w_shift = 16.* tf.random.normal([1],dtype='float32')
 else:
w_shift = 8.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def cutmix(image, label):
DIM = IMAGE_SIZE[0]
imgs = []; labs = []
for j in range(BATCH_SIZE):
P = tf.cast(tf.random.uniform([], 0, 1)<= cutmix_rate, tf.int32)
 k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)
 x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 b = tf.random.uniform([], 0, 1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image['inp1'][j,ya:yb,0:xa,:]
two = image['inp1'][k,ya:yb,xa:xb,:]
three = image['inp1'][j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image['inp1'][j,0:ya,:,:],middle,image['inp1'][j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, 1))
return {'inp1': image2, 'inp2': image['inp2']}, label2
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.7, 1.3)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<compute_train_metric>
params = {
'C': scipy.stats.expon(scale=78),
'class_weight':['balanced', None],
'gamma': scipy.stats.expon(scale=.1),
'kernel':['rbf', 'linear']
}
svc_tuned = random_search(X_train, y_train, estimator=svc, params=params )
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name = 'inp2')
 efnetb3 = efn.EfficientNetB3(weights = 'imagenet', include_top = False)
 x = efnetb3(inp1)
 x = tf.keras.layers.GlobalMaxPooling2D()(x)
 x1 = tf.keras.layers.Dense(50 )(inp2)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation('relu' )(x1)
 concat = tf.keras.layers.concatenate([x, x1])
 concat = tf.keras.layers.Dense(512, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 concat = tf.keras.layers.Dense(182, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.15 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [binary_focal_loss(gamma = 2.0, alpha = 0.80)],
metrics = [tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ] 
)
 return model
def train_and_predict(SUB, folds = 5):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 models.append(model)
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('EfficientNetB3_384.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv('submission.csv', index = False)
 print('Submission file generated')
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(submission )<define_variables>
y_pred = svc_tuned.predict(X_val)
 accuracy_svc = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_svc )
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 16,
read_size = 256,
crop_size = 250,
net_size = 224,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 5,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 12,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)<install_modules>
xgb = XGBClassifier(random_state=SEED, verbosity=0)
 xgb.get_params()
!pip install -q efficientnet<set_options>
params = {
'colsample_bytree': list(np.arange(0.6, 1.0, step=0.05)) ,
'gamma': list(np.arange(0.1, 15, step=0.2)) ,
'learning_rate': [0.01, 0.05, 0.1, 0.15, 0.21],
'max_depth': list(range(2, 12)) ,
'min_child_weight': list(range(1, 12)) ,
'n_estimators': [10, 100, 500, 1000],
'reg_alpha': [10**i for i in range(-5, 1)],
'subsample': list(np.arange(0.6, 1.0, step=0.05))
}
xgb_tuned = random_search(X_train, y_train, estimator=xgb, params=params )
random.seed(a=42)
<load_from_csv>
y_pred = xgb_tuned.predict(X_val)
 accuracy_xgboost = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_xgboost )
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )<choose_model_class>
decision_tree = DecisionTreeClassifier(random_state=SEED)
 decision_tree.get_params()
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<normalization>
params = {
'criterion': ["gini", "entropy"],
'max_depth': list(range(1, 32)) ,
'max_features': list(range(1, X_train.shape[1]+1)) ,
'min_samples_leaf': list(range(1, 9)) ,
'min_samples_split': list(np.arange(0.1, 1.1, step=0.1))
}
decision_tree_tuned = random_search(
 X_train, y_train, estimator=decision_tree, params=params )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
y_pred = decision_tree_tuned.predict(X_val)
 accuracy_decision_tree = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_decision_tree )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
knn = KNeighborsClassifier()
knn.get_params()
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<prepare_x_and_y>
params = {
'leaf_size': list(range(20, 60)) ,
'n_neighbors': list(range(3, 30)) ,
'p': [1, 2]
}
knn_tuned = random_search(X_train, y_train, estimator=knn, params=params )
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<create_dataframe>
y_pred = knn_tuned.predict(X_val)
 accuracy_knn = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_knn )
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, target:(prepare_image(img, cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )<create_dataframe>
logistic_regression = LogisticRegression(random_state=SEED)
 logistic_regression.get_params()
ds = get_dataset(files_test, CFG, labeled=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<choose_model_class>
params = {
'C': scipy.stats.loguniform(1e-4, 100),
'penalty': ['l1', 'l2', 'elasticnet'],
'solver': ['newton-cg', 'lbfgs', 'liblinear']
}
logistic_regression_tuned = random_search(
 X_train, y_train, estimator=logistic_regression, params=params )
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
y_pred = logistic_regression_tuned.predict(X_val)
 accuracy_logistic_regression = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_logistic_regression )
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 outputs = []
for i in range(cfg['net_count']):
constructor = getattr(efn, f'EfficientNetB{i}')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.summary()
return model<choose_model_class>
naive_bayes = GaussianNB()
naive_bayes.get_params()
def compile_new_model(cfg):
with strategy.scope() :
model = get_model(cfg)
 losses = [tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac'])
 for i in range(cfg['net_count'])]
model.compile(
 optimizer = cfg['optimizer'],
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model<train_model>
params = {
'var_smoothing': [np.exp(-i)for i in range(1, 15)]
}
naive_bayes_tuned = random_search(
 X_train, y_train, estimator=naive_bayes, params=params, n_iter=15-1 )
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 ds_train = ds_train.map(lambda img, label:(img, tuple([label] * CFG['net_count'])))
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS)
 model = compile_new_model(CFG)
 history = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)] )<predict_on_test>
y_pred = naive_bayes_tuned.predict(X_val)
 accuracy_naive_bayes = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_naive_bayes )
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = np.stack(probs)
 probs = probs[:,:cnt_test * CFG['tta_steps']]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=1), axis=1)
 probs = np.mean(probs, axis=1 )<create_dataframe>
voting = VotingClassifier(
 estimators=[('rf', random_forest_tuned), 
('xgb', xgb_tuned), 
('knn', knn_tuned), 
('svc', svc_tuned), 
('lr', logistic_regression_tuned), 
('dt', decision_tree_tuned), 
('nb', naive_bayes_tuned)],
voting='soft',
n_jobs=-1)
 voting = voting.fit(X_train, y_train )
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
y_pred = voting.predict(X_val)
 accuracy_voting = accuracy_score(y_val, y_pred)
 print("Accuracy:", accuracy_voting )
<save_to_csv><EOS> 
model = voting
predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("The results successfully saved!" )