Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<feature_engineering>
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 16,
read_size = 224,
crop_size = 224,
net_size = [224, 240, 260, 300, 380, 456, 528, 600],
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 5,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 20,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)
data['AgeBins'] = 0<feature_engineering>
!pip install -q efficientnet
data['AgeBins'].loc[(data['Age'] >= 11)&(data['Age'] < 20)] = 1
data['AgeBins'].loc[(data['Age'] >= 20)&(data['Age'] < 60)] = 2
data['AgeBins'].loc[data['Age'] >= 60] = 3<feature_engineering>
!pip install missingpy
data['FareBins'] = pd.qcut(data['Fare'], 4 )<drop_column>
random.seed(a=42)
data.drop(columns=['PassengerId','Name','Ticket', 'Cabin', 'Age', 'Fare', 'SibSp', 'Parch'], inplace= True )<categorify>
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
data = pd.get_dummies(
 data, columns=['Embarked', 'Sex', 'Title'], drop_first=True )<categorify>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
label = LabelEncoder()
data['FareBins'] = label.fit_transform(data['FareBins'] )<define_variables>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
labelled = data[data.Survived.isnull() == False].reset_index(drop=True)
 unlabelled = data[data.Survived.isnull() ].drop(columns = ['Survived'] ).reset_index(drop=True )<data_type_conversions>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']), example['target'])
 def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']),(example['image_name'] if return_image_name else 0))
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 return img
def prepare_data(data, cfg=None, augment=True):
img = data[0]
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'][0], cfg['net_size'][0]])
 img = tf.reshape(img, [cfg['net_size'][0], cfg['net_size'][0], 3])
 sex_oh = tf.one_hot(data[1][0], 2)
 age_aprox = tf.dtypes.cast(tf.reshape(data[1][1], [1]), tf.float32)
 anatom_site_general_challenge = tf.one_hot(data[1][2], 7)
 dense = tf.concat([sex_oh, age_aprox, anatom_site_general_challenge], axis=0)
 return(img, dense)
 def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
labelled['Survived'] = labelled.Survived.astype('int64' )<normalization>
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, imgname_or_label:(prepare_data(( img, dense), augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
scalers = [MinMaxScaler() , MaxAbsScaler() , StandardScaler() , RobustScaler() ,
Normalizer() , QuantileTransformer() , PowerTransformer() ]<train_on_grid>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data[0][0], data[1]
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
scaler_score = {}
labelled_copy = labelled.copy(deep= True)
 for scaler in scalers:
scaler.fit(labelled_copy[['FamilySize']])
 labelled_copy['FamilySize'] = scaler.transform(labelled_copy[['FamilySize']])
 lr = LogisticRegressionCV(cv = 10, scoring= 'accuracy')
 lr.fit(labelled_copy.drop(columns=['Survived']), labelled_copy.Survived)
 score = lr.score(labelled_copy.drop(columns=['Survived']), labelled_copy.Survived)
 scaler_score.update({scaler:score} )<normalization>
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, target:(prepare_data(( img, dense), cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )
scaler = StandardScaler()
scaler.fit(labelled[['FamilySize']])
 labelled['FamilySize'] = scaler.transform(labelled[['FamilySize']])
 unlabelled['FamilySize'] = scaler.transform(unlabelled[['FamilySize']] )<split>
ds = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
x_train, x_other, y_train, y_other = train_test_split(
 labelled.drop(columns=['Survived']), labelled.Survived, train_size=0.7 )<split>
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
x_valid, x_test, y_valid, y_test = train_test_split(
 x_other, y_other, train_size=0.5 )<prepare_x_and_y>
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'][0], cfg['net_size'][0], 3), name='imgIn')
 model_input_data = tf.keras.Input(shape=(10,), name='dataIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 dense = tf.keras.layers.Dense(1024, activation='relu' )(model_input_data)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 dense = tf.keras.layers.Dense(256, activation='relu' )(dense)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 outputs = []
for i in range(cfg['net_count']):
constructor = getattr(efn, f'EfficientNetB{i}')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'][i], cfg['net_size'][i], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Concatenate()([x, dense])
 x = tf.keras.layers.Dense(256, activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.5 )(x)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model([model_input, model_input_data], outputs, name='aNetwork')
 model.summary()
return model
features = labelled.drop(columns=['Survived'])
 target = labelled.Survived<choose_model_class>
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
logistic_reg = LogisticRegressionCV(cv= 7 )<define_search_space>
def compile_new_model(cfg, old_model=None):
with strategy.scope() :
if(old_model is None):
model = get_model(cfg)
 else:
model = old_model
losses = [tfa.losses.SigmoidFocalCrossEntropy(gamma = 2.0, alpha = 0.80)for i in range(cfg['net_count'])]
lr = tf.keras.optimizers.schedules.ExponentialDecay(
 initial_learning_rate=0.1,
decay_steps=10000,
decay_rate=0.9)
 model.compile(
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.1),
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model
threshold = np.arange(1, 10, 0.5)*1e-1<compute_test_metric>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 class_weights = class_weight.compute_class_weight('balanced',
np.unique(train.target),
train.target)
 class_weights = dict(enumerate(class_weights))
print('The highest accuracy score is:', np.max(np.array(scores)) )<define_variables>
file_path=".. /input/output/effnet_comb_weights.best.hdf5"
number_of_features = list(range(1,13))<compute_test_metric>
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS )
print("Maximum accuracy score is :", max(scores_k))<define_variables>
print("Buidling model...")
 model = compile_new_model(CFG )
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )<define_variables>
print("Disabling effnets for training...")
 for i in range(CFG['net_count']):
for layer in model.layers:
if(layer.name==f'efficientnet-b{i}'):
layer.trainable = False
break
print("Rebuilding model")
 model = compile_new_model(CFG, model)
 history = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = 5,
class_weight=class_weights )
print("Optimal number of features : %d" % selector.n_features_ )<compute_test_metric>
print("Disabling effnets for training...")
 for i in range(CFG['net_count']):
for layer in model.layers:
if(layer.name==f'efficientnet-b{i}'):
layer.trainable = True
break
model = compile_new_model(CFG, model)
 history_complete = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)],
class_weight=class_weights )
print("Maximum accuracy score is :", np.max(selector.grid_scores_))<define_search_space>
threshold = np.arange(1, 5, 0.1)*1e-1<compute_test_metric>
df_train.sex.isna().sum()
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )<compute_test_metric>
df_train.anatom_site_general_challenge.isna().sum()
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )<features_selection>
df_train.age_approx.isna().sum()
selector = sklearn.feature_selection.SelectFromModel(logistic_reg, threshold= 0.25)
 selector.fit(features, target)
 lr_selected_features = selector.get_support()<choose_model_class>
df_train['sex'] = pd.Categorical(df_train.sex)
 df_train['sex_cat'] = df_train.sex.cat.codes
df_train['anatom_site_general_challenge'] = pd.Categorical(df_train.anatom_site_general_challenge)
 df_train['anatom_site_general_challenge_cat'] = df_train.anatom_site_general_challenge.cat.codes
logistic_reg = LogisticRegressionCV(
 Cs=1, cv= 7, scoring='accuracy', max_iter=1000, refit=True )<init_hyperparams>
imputer = MissForest()
df_train[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']] = imputer.fit_transform(df_train[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']] )
lr_parameters_1 = {'solver': ['liblinear', 'saga'], 'penalty': ['l1']}
lr_parameters_2 = {'solver': ['newton-cg', 'lbfgs', 'sag'], 'penalty': ['l2']}<choose_model_class>
print("Sex nan:", df_train.sex_cat.isna().sum())
 print("anatom_site_general_challenge nan:", df_train.anatom_site_general_challenge_cat.isna().sum())
 print("age_approx nan:", df_train.age_approx.isna().sum() )
rs_lr = RandomizedSearchCV(logistic_reg, param_distributions= lr_parameters_2, n_iter= 100 )<train_model>
train, test = train_test_split(df_train, test_size=0.2)
 print(len(train))
print(len(test))
X_train, y_train = train[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']], train.target
X_test, y_test = test[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']], test.target
rs_lr.fit(x_train.loc[:, lr_selected_features], y_train )<train_model>
class_weights = list(class_weight.compute_class_weight('balanced',
np.unique(y_train),
y_train))
weights_train = np.ones(y_train.shape[0], dtype = 'float')
 for i, val in enumerate(y_train):
weights_train[i] = class_weights[val-1]
weights_test = np.ones(y_test.shape[0], dtype = 'float')
 for i, val in enumerate(y_test):
weights_test[i] = class_weights[val-1]
print('Best Parameters are:
', rs_lr.best_params_,
'
Training accuracy score is:
', rs_lr.best_score_ )<compute_test_metric>
train_mat = xgb.DMatrix(X_train, y_train, weight=weights_train)
 test_mat = xgb.DMatrix(X_test, y_test, weight=weights_test )
print('Validation accuracy score is:
', rs_lr.score(
 x_valid.loc[:, lr_selected_features], y_valid))<train_on_grid>
class XGBoostClassifier(xgb.XGBClassifier):
def __init__(self, num_boost_round=10, **params):
self.clf = None
self.num_boost_round = num_boost_round
if 'num_boost_round' in params:
self.num_boost_round = params.pop('n_estimators')
 self.params = params
self.params.update({'objective': 'binary:logistic'})
 def fit(self, X, y, num_boost_round=None, weights=None):
num_boost_round = num_boost_round or self.num_boost_round
dtrain = xgb.DMatrix(X, y, weights)
 self.clf = xgb.train(params=self.params, dtrain=dtrain, num_boost_round=num_boost_round)
 def predict(self, X):
return self.clf.predict(xgb.DMatrix(X))
def predict_proba(self, X):
dtest = xgb.DMatrix(X)
 classone_probs = self.clf.predict(dtest)
 classzero_probs = 1.0 - classone_probs
return np.vstack(( classzero_probs, classone_probs)).transpose()
def get_params(self, deep=True):
return self.params
def set_params(self, **params):
if 'n_estimators' in params:
self.num_boost_round = params.pop('n_estimators')
 self.params.update(params)
 return self
param_name = 'Cs'
param_range = [1, 10, 100, 1000]
train_score, valid_score = [], []
for cs in param_range:
lr = LogisticRegressionCV(Cs=cs, cv=7, scoring='accuracy', solver= 'newton-cg',
penalty= 'l2', refit=True, max_iter=1000)
 lr.fit(x_train.loc[:, lr_selected_features], y_train)
 train_score.append(
 lr.score(x_train.loc[:, lr_selected_features], y_train))
valid_score.append(
 lr.score(x_valid.loc[:, lr_selected_features], y_valid))<train_on_grid>
params = {
'max_depth':[3, 5, 10],
'learning_rate': [0.5, 0.1, 0.001],
'n_estimators': [300, 500, 1500],
'subsample': [0.5, 0.7, 1.0],
'colsample_bytree': [0.01, 0.75, 1]
}
xgbc = XGBoostClassifier(objective='binary:logistic')
 clf = GridSearchCV(xgbc, params, scoring='roc_auc', cv=3)
 clf.fit(X_train, y_train, weights=weights_train )
lr = LogisticRegressionCV(Cs= 10, cv= 7, solver= 'newton-cg', penalty= 'l2')
 lr.fit(x_train.loc[:, lr_selected_features], y_train )<compute_train_metric>
params = {
'objective':'binary:logistic',
'colsample_bytree': 0.75,
'learning_rate': 0.001,
'max_depth': 13,
'n_estimators': 1500,
'subsample': 0.3}
bst = xgb.train(clf.best_params_, train_mat, clf.best_params_['n_estimators'])
 roc_auc_score(y_train, bst.predict(train_mat))
y_scores_lr = lr.predict_proba(x_test.loc[:, lr_selected_features])[:, 1]
lr_fpr, lr_tpr, lr_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_lr )<choose_model_class>
roc_auc_score(y_test, bst.predict(test_mat))
lr_auc = sklearn.metrics.auc(x=lr_fpr, y=lr_tpr )<compute_test_metric>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = np.stack(probs)
 probs = probs[:,:cnt_test * CFG['tta_steps']]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=1), axis=1)
 probs = np.mean(probs, axis=1 )
lr_acc = lr.score(x_test.loc[:, lr_selected_features], y_test )<compute_test_metric>
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
print('For logistic Regression:
Area Under Curve: {},
Test Accuracy score: {}'.format(
 lr_auc, lr_acc))<choose_model_class>
for i in range(1):
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission_model_{i}.csv', index=False )
nb = GaussianNB()<define_search_space>
submission = pd.DataFrame(dict(
 image_name = image_names,
target = np.mean(probs[:,:,0], axis=0)))
 submission = submission.sort_values('image_name')
 submission.to_csv('submission_models_blended.csv', index=False )
threshold = np.arange(1, 10, 0.5)*1e-1<compute_train_metric>
df_test['sex'] = pd.Categorical(df_test.sex)
 df_test['sex_cat'] = df_test.sex.cat.codes
df_test['anatom_site_general_challenge'] = pd.Categorical(df_test.anatom_site_general_challenge)
 df_test['anatom_site_general_challenge_cat'] = df_test.anatom_site_general_challenge.cat.codes
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 nb.fit(selected_features, target)
 y_pred = nb.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(np.arange(1, 10, 0.5)*1e-1, np.array(scores))<compute_test_metric>
imputer = MissForest()
df_test[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']] = imputer.fit_transform(df_test[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']] )
print('The highest accuracy score is:', np.max(np.array(scores)) )<define_variables>
X_test = df_test[['sex_cat', 'age_approx', 'anatom_site_general_challenge_cat']]
number_of_features = list(range(1,13))<compute_test_metric>
df_test['xgb_pred'] = bst.predict(xgb.DMatrix(X_test))
df_test = df_test.set_index('image_name' )
print("Maximum accuracy score is :", max(scores_k))<define_variables>
pred_df = pd.DataFrame(probs[:,:,0] ).transpose()
pred_df['image_name'] = image_names
pred_df = pred_df.set_index('image_name')
 pred_df = pd.concat([pred_df, df_test['xgb_pred']], axis=1 )
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )<train_on_grid>
df_test[df_test.index=='ISIC_1581247']
selector = sklearn.feature_selection.VarianceThreshold(threshold= 0.1)
 selector.fit(features, target)
 nb_selected_features = selector.get_support()<define_search_space>
pred_df['final_prediction'] = np.average(pred_df[list(range(7)) ].values, axis=1)
 pred_df['final_prediction'] = pred_df['final_prediction']*0.95+pred_df['xgb_pred']*0.05
<choose_model_class><EOS> 
pred_df = pred_df.sort_index()
pred_df['target'] = pred_df['final_prediction']
pred_df['image_name'] = pred_df.index
pred_df[['image_name', 'target']].to_csv('submission_models_blended_with_dense.csv', index=False )