Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<split>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(10,4))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(10,4))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
X=train.drop(["Survived","PassengerId"],axis=1)
 y=train["Survived"]
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=100 )<predict_on_test>
NAME='EffNB6_512'
NFOLDS=5
NBEST=1
SEED=311
ef = 3
DROPOUT = True
droprate=0.5
dropct= 4
dropsize=0.2
TTA = 3
EPOCHS = 10
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [384,384]
ANATOM = 1
dim = IMAGE_SIZE[0]
DIM = dim
AUTO = tf.data.experimental.AUTOTUNE
k = IMAGE_SIZE[0]
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
GCS_PATH3 = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i'%(k,k))
USE_EXT_DATA = True
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
ids=test["PassengerId"]
Xtest=test.drop(["PassengerId"],axis=1)
 pred_1=lr.predict(Xtest)
 pred_2=ranF.predict(Xtest)
 pred_3=abc.predict(Xtest)
 pred_4=gbc.predict(Xtest)
 pred_5=clf.predict(Xtest)
 pred=[]
for i in range(len(pred_1)) :
l_0=1
l_1=1
if(pred_1[i]==0):
l_0*=p1[0,0]
l_1*=p1[1,0]
else:
l_0*=p1[0,1]
l_1*=p1[1,1]
if(pred_2[i]==0):
l_0*=p2[0,0]
l_1*=p2[1,0]
else:
l_0*=p2[0,1]
l_1*=p2[1,1]
if(pred_3[i]==0):
l_0*=p3[0,0]
l_1*=p3[1,0]
else:
l_0*=p3[0,1]
l_1*=p3[1,1]
if(pred_4[i]==0):
l_0*=p4[0,0]
l_1*=p4[1,0]
else:
l_0*=p4[0,1]
l_1*=p4[1,1]
if(pred_5[i]==0):
l_0*=p4[0,0]
l_1*=p4[1,0]
else:
l_0*=p4[0,1]
l_1*=p4[1,1]
if(l_0>l_1):
pred.append(0)
 else:
pred.append(1 )<concatenate>
def append_path(pre):
return np.vectorize(lambda file: os.path.join(GCS_DS_PATH, pre, file))
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 sns.countplot(train['target'] )
Survived=pd.Series(pred_5)
 output=pd.concat([ids,Survived],axis=1)
 output.columns=["PassengerId","Survived"]<save_to_csv>
%%time
ALL_TRAIN=tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 ALL_TRAIN1 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 ALL_TRAIN2 = tf.io.gfile.glob(GCS_PATH3 + '/train*.tfrec')
 ALL_TRAIN
output.to_csv("submission.csv",index=False)
 output.head()<set_options>
tot_train = len(ALL_TRAIN)
 skip = tot_train//NFOLDS
VAL_FNAMES={}
x = 0
for n in range(1, NFOLDS+1):
VAL_FNAMES[f"fold_{n}"] = ALL_TRAIN[x : x+skip]
x += skip
if USE_EXT_DATA :
ALL_TRAIN += ALL_TRAIN1
ALL_TRAIN += ALL_TRAIN2
TRAIN_FNAMES={f'fold_{i}': list(set(ALL_TRAIN)-set(VAL_FNAMES[f'fold_{i}']))
for i in range(1, NFOLDS+1)}
TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec' )
np.random.seed(42 )<load_from_csv>
print(f"total folds : {len(TRAIN_FNAMES)} Total per fold : {len(TRAIN_FNAMES['fold_1'])}" )
train_data = pd.read_csv('/kaggle/input/titanic/train.csv', index_col=0)
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv',index_col=0)
 train_data.head()<prepare_x_and_y>
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
X = train_data.drop(columns=['Survived'])
 y = train_data.Survived<count_values>
if colab:
PATH=Path('/content/gdrive/My Drive/kaggle/input/siim-isic-melanoma-classification/')
 train=pd.read_csv(PATH/'train.csv.zip')
 else:
PATH=Path('/kaggle/input/siim-isic-melanoma-classification/')
 train=pd.read_csv(PATH/'train.csv')
 test=pd.read_csv(PATH/'test.csv')
 sub=pd.read_csv(PATH/'sample_submission.csv')
 seed_everything(SEED )
train_data[train_data.Age <= 15].Survived.value_counts(normalize=True )<count_values>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
print(train_data[train_data.Sex == 'male'].Survived.value_counts(normalize=True))
print('-'*40)
 print(train_data[train_data.Sex == 'female'].Survived.value_counts(normalize=True))<groupby>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name':tf.io.FixedLenFeature([], tf.string),
'patient_id': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'target': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 data = {}
dumpp = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['image_name'] = tf.cast(example['image_name'], tf.string)
 if ANATOM :
dumpp['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 for i in range(7):
data[f'anatom_site_general_challenge{i}'] = dumpp['anatom_site_general_challenge'][i]
return image, label ,data , label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name':tf.io.FixedLenFeature([], tf.string),
'patient_id': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 data = {}
dumpp = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['image_name'] = tf.cast(example['image_name'], tf.string)
 if ANATOM :
dumpp['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 for i in range(7):
data[f'anatom_site_general_challenge{i}'] = dumpp['anatom_site_general_challenge'][i]
return image , data
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled
else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
train_data.groupby('Pclass' ).Survived.value_counts(normalize=True, sort=False )<count_values>
ranks = X['Name'].str.extract(r'\b(\w+)\.')
 ranks[0].value_counts()<count_missing_values>
ranks[train_data.Age.isna() ][0].value_counts()<count_missing_values>
def dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
missing_age_ranks = ranks[train_data.Age.isna() ][0].value_counts().index.values
missing_age_ranks<feature_engineering>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3])
 def data_augment(data, label):
img = data['inp1']
img = transform(img,DIM=dim)
 if DROPOUT :
img = dropout(img, DIM=dim, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 data['inp1'] = img
return data, label
def data_augment_test(data , label = 0):
img = data['inp1']
img = transform(img,DIM=dim)
 if DROPOUT :
img = dropout(img, DIM=dim, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 data['inp1'] = img
return data , 0
age_na_fills = {}
for rank in missing_age_ranks:
age_na_fills[rank] = round(train_data[(ranks == rank ).values].Age.mean())
 age_na_fills<data_type_conversions>
LR_START = 0.000005
LR_MAX = 0.00000725 * strategy.num_replicas_in_sync
LR_MIN = 0.000005
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 4
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
def fill_age_na(data, age_na_fills, ranks):
data['Age'] = data.apply(lambda row: age_na_fills.get(ranks[row.name], 29)if np.isnan(row['Age'])else row['Age'], axis=1 )<categorify>
train['target'].values
class_weights = class_weight.compute_class_weight(class_weight='balanced',
classes=np.unique(train['target'].values),
y=train['target'].values, 
)
 class_weights = {i : class_weights[i] for i in range(len(class_weights)) }
print(class_weights )
class AgeFillNa(BaseEstimator, TransformerMixin):
def __init__(self, age_na_fills=None):
self.age_na_fills = age_na_fills
def fit(self, X, y=None):
return self
def transform(self, X):
output = X.copy()
ranks = X['Name'].str.extract(r'\b(\w+)\.')
 if self.age_na_fills is not None:
fill_age_na(output, self.age_na_fills, ranks[0])
 else:
output.Age = output.Age.fillna(output.Age.mean())
 return output
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, columns_drop):
self.columns_drop = columns_drop
def fit(self, X, y=None):
return self
def transform(self, X):
return X.drop(columns=self.columns_drop)
 class MultiColumnLabelEncoder(BaseEstimator, TransformerMixin):
def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y=None):
return self
def transform(self, X):
output = X.copy()
if self.columns is not None:
for col in self.columns:
output[col] = LabelEncoder().fit_transform(output[col].astype(str))
else:
for colname, col in output.iteritems() :
output[colname] = LabelEncoder().fit_transform(col)
 return output
columns_to_drop = ['Ticket', 'Cabin', 'Name']
columns_to_encode = ['Sex', 'Embarked']<categorify>
if ANATOM :
tab_feats=[
'sex',
'age_approx',
'anatom_site_general_challenge0',
'anatom_site_general_challenge1',
'anatom_site_general_challenge2',
'anatom_site_general_challenge3',
'anatom_site_general_challenge4',
'anatom_site_general_challenge5',
'anatom_site_general_challenge6'
]
else :
tab_feats=[
'sex',
'age_approx',
]
N_TAB_FEATS=len(tab_feats)
 print(f"The number of tabular features is {N_TAB_FEATS}." )
prepare_pipeline = Pipeline([ 
('age_fill_na', AgeFillNa(age_na_fills)) , 
('selector', DataFrameSelector(columns_to_drop)) , 
('label_enc', MultiColumnLabelEncoder(columns=columns_to_encode)) , 
('imputer', SimpleImputer(strategy="mean")) , 
('std_scaler', StandardScaler()),
] )<normalization>
%time
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def get_model() :
with strategy.scope() :
pretrained_model = EFNS[ef](input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 pretrained_model.trainable = True
inp1 = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name='inp1')
 inp2 = tf.keras.layers.Input(shape=(N_TAB_FEATS), name='inp2')
 x=pretrained_model(inp1)
 x=tf.keras.layers.GlobalAveragePooling2D()(x)
 x=tf.keras.layers.Dense(512,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(256,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(128,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 y=tf.keras.layers.Dense(100,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(inp2)
 concat=tf.keras.layers.concatenate([y, x])
 output = tf.keras.layers.Dense(1, activation='sigmoid', name='output' )(concat)
 model = tf.keras.models.Model(inputs=[inp1,inp2], outputs=[output])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=[tf.keras.metrics.AUC() ], 
)
 return model
X = prepare_pipeline.fit_transform(train_data.drop(columns=['Survived']))<predict_on_test>
if colab:
SAVE_FOLDER=NAME
if tstamp:
time_zone = pytz.timezone('America/Chicago')
 current_datetime = datetime.now(time_zone)
 ts=current_datetime.strftime("%m%d%H%M%S")
 SAVE_FOLDER+='_'+ts
SAVE_FOLDER=PATH/SAVE_FOLDER
if not os.path.exists(SAVE_FOLDER):
os.mkdir(SAVE_FOLDER)
 else:
SAVE_FOLDER=Path('/kaggle/working' )
empt_models = [GradientBoostingClassifier() , LogisticRegression(solver='lbfgs'), SGDClassifier() , GaussianNB() ,
KNeighborsClassifier() , DecisionTreeClassifier() , RandomForestClassifier(n_estimators=100),
SVC(gamma='auto')]
for model in empt_models:
results = cross_val_predict(model, X, y, cv=5)
 print("model: {}, result: {:.3f}".format(model.__class__, f1_score(y, results)) )<import_modules>
class save_best_n(tf.keras.callbacks.Callback):
def __init__(self, fn, model):
self.fn = fn
self.model = model
def on_epoch_end(self, epoch, logs=None):
if(epoch>0):
score=logs.get("val_auc")
 else:
score=-1
if(score > best_score[fold_num].min()):
idx_min=np.argmin(best_score[fold_num])
 best_score[fold_num][idx_min]=score
best_epoch[fold_num][idx_min]=epoch+1
path_best_model=f'best_model_fold_{self.fn}_{idx_min}.hdf5'
self.model.save(SAVE_FOLDER/path_best_model)
from sklearn.model_selection import GridSearchCV<import_modules>
def setup_input(image, label, data, label_name):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}, label
from sklearn.model_selection import GridSearchCV<train_on_grid>
def get_training_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
params = [{'kernel':['linear'], 'gamma':[i / 10 for i in range(1, 100, 5)], 'C':[i / 10 for i in range(5, 30, 5)]},
{'kernel':['rbf'], 'C':[i / 10 for i in range(5, 30, 5)]},
{'kernel':['poly'], 'degree':np.arange(2, 10), 'C':[i / 10 for i in range(5, 30, 5)]}]
grid_search_svc = GridSearchCV(SVC(gamma='auto'), params, cv=5, scoring='neg_mean_squared_error')
 grid_search_svc.fit(X, y)
 grid_search_svc.best_params_<compute_train_metric>
def get_validation_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
svc_model = SVC(gamma='auto', **grid_search_svc.best_params_)
 svc_score = cross_val_predict(svc_model, X, y, cv=5)
 f1_score(y, svc_score )<train_on_grid>
train_dataset = load_dataset(TRAIN_FNAMES['fold_1'])
 train_dataset
params = {'learning_rate':[i/10 for i in range(3, 6)], 'n_estimators':np.arange(128, 132, 1),
'min_samples_split':np.arange(2, 3), 'min_samples_leaf':np.arange(6, 8),
'max_depth':np.arange(1, 3), 'max_features':np.arange(1, 8)}
grid_search_gbc = GridSearchCV(GradientBoostingClassifier() , params, cv=5, scoring='neg_mean_squared_error')
 grid_search_gbc.fit(X, y)
 grid_search_gbc.best_params_<compute_train_metric>
debug=0
histories = []
best_epoch={fn: np.zeros(NBEST)for fn in range(1, NFOLDS+1)}
best_score={fn: np.zeros(NBEST)for fn in range(1, NFOLDS+1)}
for fold_num in range(1, NFOLDS+1):
tf.keras.backend.clear_session()
tf.tpu.experimental.initialize_tpu_system(tpu)
 print("="*50)
 print(f"Starting fold {fold_num} out of {NFOLDS}...")
 files_trn=TRAIN_FNAMES[f"fold_{fold_num}"]
files_val=VAL_FNAMES[f"fold_{fold_num}"]
if debug:
NUMS = 2
files_trn=files_trn[0:2]
files_val=files_val[0:2]
EPOCHS=2
train_dataset = load_dataset(files_trn)
 train_dataset = train_dataset.map(setup_input, num_parallel_calls=AUTO)
 val_dataset = load_dataset(files_val, ordered = True)
 val_dataset = val_dataset.map(setup_input, num_parallel_calls=AUTO)
 model = get_model()
STEPS_PER_EPOCH = count_data_items(files_trn)// BATCH_SIZE
print(f'STEPS_PER_EPOCH = {STEPS_PER_EPOCH}')
 lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 history = model.fit(get_training_dataset(train_dataset),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback,
save_best_n(fold_num, model),
],
validation_data=get_validation_dataset(val_dataset),
class_weight=class_weights,
verbose=2, 
)
 idx_sorted=np.argsort(best_score[fold_num])
 best_score[fold_num]=np.array(best_score[fold_num])[idx_sorted]
best_epoch[fold_num]=np.array(best_epoch[fold_num])[idx_sorted]
print(f"
Fold {fold_num} is finished.The best epochs: {[int(best_epoch[fold_num][i])for i in range(len(best_epoch[fold_num])) ]}")
 print(f"The corresponding scores: {[round(best_score[fold_num][i], 5)for i in range(len(best_epoch[fold_num])) ]}")
 histories.append(history )
gbc_model = GradientBoostingClassifier(**grid_search_gbc.best_params_)
 gbc_score = cross_val_predict(gbc_model, X, y, cv=5)
 f1_score(y, gbc_score )<train_on_grid>
train.to_csv('train.csv' )
params = {'n_neighbors':np.arange(10, 25), 'p':np.arange(1, 6)}
grid_search_knn = GridSearchCV(KNeighborsClassifier() , params, cv=5, scoring='neg_mean_squared_error')
 grid_search_knn.fit(X, y)
 grid_search_knn.best_params_<compute_train_metric>
history.history
knn_model = KNeighborsClassifier(**grid_search_knn.best_params_)
 knn_score = cross_val_predict(knn_model, X, y, cv=5)
 f1_score(y, knn_score )<train_on_grid>
lsti = []
TEST_FNAMES = TEST_FILENAMES
for i in range(TTA):
lstj = []
N_TEST_IMGS = count_data_items(TEST_FNAMES)
 preds = pd.DataFrame({'image_name': np.zeros(len(test)) , 'target': np.zeros(len(test)) })
 test_ds = load_dataset(TEST_FNAMES, labeled=False, ordered=True)
 test_images_ds = test_ds.map(setup_test_image, num_parallel_calls=AUTO)
 test_images_ds = get_test_dataset(test_images_ds)
 test_ds = get_test_ds_name(test_ds)
 test_ids_ds = test_ds.map(setup_test_name, num_parallel_calls=AUTO ).unbatch()
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 preds['image_name'] = next(iter(test_ids_ds.batch(N_TEST_IMGS)) ).numpy().astype('U')
 lstj = np.average([average_predictions(test_images_ds, fn)for fn in range(1, NFOLDS + 1)], axis = 0)
 lsti.append(lstj)
 del test_images_ds , test_ds , test_ids_ds
gc.collect()
lsti
params = {'n_estimators':np.arange(50, 1000, 100), 'max_depth':np.arange(3, 8)}
grid_search_rfc = GridSearchCV(RandomForestClassifier(random_state=42), params, cv=5, scoring='neg_mean_squared_error')
 grid_search_rfc.fit(X, y)
 grid_search_rfc.best_params_<compute_train_metric>
a = np.array(lsti)
 a
rfc_model = RandomForestClassifier(random_state=42,**grid_search_rfc.best_params_)
 rfc_score = cross_val_predict(rfc_model, X, y, cv=5)
 f1_score(y, rfc_score )<categorify>
preds['target'] = a.mean(axis=0 )
<predict_on_test><EOS> 
%%time
del sub['target']
sub = sub.merge(preds, on='image_name')
 sub.head()
sub.to_csv('effB1_512_5x15epochs_3Stratif.csv', index=False)