Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<save_to_csv>
!pip install -q efficientnet
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.Label = y[arg_min_dists.cpu() ]
sub.to_csv('sub.csv', index=False)
 !head sub.csv<import_modules>
import os
import re
import numpy as np
import pandas as pd
import math
from matplotlib import pyplot as plt
from sklearn import metrics
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
from fastai import *
from fastai.vision import *
from fastai.metrics import accuracy,error_rate<load_from_csv>
TRAINING = False
class CustomImageItemList(ImageList):
def open(self, fn):
img = fn.reshape(28,28)
 img = np.stack(( img,)*3, axis=-1)
 return Image(pil2tensor(img, dtype=np.float32))
@classmethod
def from_csv_custom(cls, path:PathOrStr, csv_name:str, imgIdx:int=1, header:str='infer', **kwargs)->'ItemList':
df = pd.read_csv(Path(path)/csv_name, header=header)
 res = super().from_df(df, path=path, cols=0, **kwargs)
 res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1 ).values
return res<define_variables>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
path = '.. /input'<load_from_csv>
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 EPOCHS = 5
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [1024, 1024]
test = CustomImageItemList.from_csv_custom(path=path, csv_name='test.csv', imgIdx=0)
 data =(CustomImageItemList.from_csv_custom(path=path, csv_name='train.csv')
.split_by_rand_pct (.2)
.label_from_df(cols='label')
.add_test(test, label=0)
.databunch(bs=64, num_workers=0)
.normalize(imagenet_stats))<choose_model_class>
def append_path(pre):
return np.vectorize(lambda file: os.path.join(GCS_DS_PATH, pre, file))
learn = cnn_learner(data, models.resnet50, metrics=error_rate,model_dir="/tmp/model/" )<train_model>
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec' )
learn.fit_one_cycle(4 )<save_model>
VALIDATION_FILENAMES = TRAINING_FILENAMES[int(0.8*len(TRAINING_FILENAMES)) :]
TRAINING_FILENAMES = TRAINING_FILENAMES[:int(0.8*len(TRAINING_FILENAMES)) ]
learn.save("model_1", return_path=True )<train_model>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))
learn.fit_one_cycle(1 )<load_pretrained>
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
learn.load('model_1' )<find_best_params>
if TRAINING:
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model.summary()
learn.lr_find()<train_model>
if TRAINING:
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB0(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model2.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model2.summary()
learn.unfreeze()
learn.fit_one_cycle(10, max_lr=slice(1e-5,1e-4))<save_to_csv>
if TRAINING:
with strategy.scope() :
dnet201 = DenseNet201(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 dnet201.trainable = True
model3 = tf.keras.Sequential([
dnet201,
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model3.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model3.summary()
predictions, *_ = learn.get_preds(DatasetType.Test)
 labels = np.argmax(predictions, 1)
 submission_df = pd.DataFrame({'ImageId': list(range(1,len(labels)+1)) , 'Label': labels})
 submission_df.to_csv(f'submission.csv', index=False )<set_options>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1 )
%matplotlib inline 
<define_variables>
train_dataset = get_training_dataset()
valid_dataset = get_validation_dataset()
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<load_from_csv>
if TRAINING:
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset 
)
 model.save("efficientnetb7.h5" )
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )<load_from_csv>
if TRAINING:
history2 = model2.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset 
)
 model2.save("efficientnetb0.h5" )
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )<data_type_conversions>
sub1 = sub.copy()
sub2 = sub.copy()
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )<data_type_conversions>
if TRAINING:
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures<merge>
if TRAINING:
pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})
 pred_df2 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities2)})
 del sub1['target']
sub1 = sub1.merge(pred_df, on='image_name')
 sub1.to_csv('submission_efficientnetb7.csv', index=False)
 del sub2['target']
sub2 = sub2.merge(pred_df2, on='image_name')
 sub2.to_csv('submission_efficientnetb0.csv', index=False )
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df<merge>
if TRAINING:
sub_es = sub1[['image_name']]
sub_es['target'] = 0.5*sub1['target'] + 0.5*sub2['target']
sub_es.to_csv('submission_blend.csv', index=False)
 sub_es.head()
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df<merge>
import seaborn as sns
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df<feature_engineering>
sub_path = ".. /input/siimisic-submission-files"
all_files = os.listdir(sub_path)
 all_files
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))<compute_test_metric>
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "target" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )<merge>
concat_sub['target_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['target_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['target_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['target_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )<drop_column>
cutoff_lo = 0.002
cutoff_hi = 0.11
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full<define_variables>
concat_sub['target'] = concat_sub['target_mean']
concat_sub[['image_name', 'target']].to_csv('submission_mean.csv',
index=False, float_format='%.6f' )
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]<load_from_csv>
concat_sub['target'] = concat_sub['target_median']
concat_sub[['image_name', 'target']].to_csv('submission_median.csv',
index=False, float_format='%.6f' )
%%time
full = build_couple_dataframe(train_csv, structures_csv, '1JHN', n_atoms=10)
 print(full.shape )<split>
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['target_median']))
concat_sub[['image_name', 'target']].to_csv('submission_pushout_median.csv',
index=False, float_format='%.6f' )
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 X_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.3, random_state=128)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape<prepare_x_and_y>
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['target_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['target_min'],
concat_sub['target_mean']))
concat_sub[['image_name', 'target']].to_csv('submission_minmax_mean.csv',
index=False, float_format='%.6f' )
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data<find_best_model_class>
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['target_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['target_min'],
concat_sub['target_median']))
concat_sub[['image_name', 'target']].to_csv('submission_minmax_median.csv',
index=False, float_format='%.6f' )
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, n_folds=3, n_splits=5, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 cv_score = 0
if n_folds > n_splits:
n_splits = n_folds
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
if fold >= n_folds:
break
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
iterations=3000
if coupling_type in ['1JHN','1JHC']:
iterations=12000
model = CatBoostRegressor(n_estimators=iterations,
verbose=-1,
objective="MAE",
loss_function="MAE",
boosting_type="Ordered",
reg_lambda=0,
learning_rate=0.2,
task_type='GPU',
subsample=0.9,
max_depth=10,
bootstrap_type='Bernoulli',
colsample_bylevel=1 
)
 model.fit(X_train, y_train,
eval_set=[(X_val, y_val)],
verbose=100,
early_stopping_rounds=300)
 y_val_pred = model.predict(X_val)
 val_score = np.log(mean_absolute_error(y_val, y_val_pred))
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(X_test)/ n_folds
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
return cv_score<load_from_csv>
!pip install -q efficientnet
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )<find_best_params>
import os
import re
import numpy as np
import pandas as pd
import math
from matplotlib import pyplot as plt
from sklearn import metrics
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
N_FOLDS = 5
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score<save_to_csv>
TRAINING = False
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
%matplotlib inline 
<load_from_csv>
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 EPOCHS = 5
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [1024, 1024]
test = pd.read_csv(".. /input/test.csv")
 train = pd.read_csv(".. /input/train.csv" )<prepare_x_and_y>
def append_path(pre):
return np.vectorize(lambda file: os.path.join(GCS_DS_PATH, pre, file))
y_train = train["label"]
y_train.head()<drop_column>
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec' )
x_train = train.drop(labels = ["label"],axis = 1 )<count_values>
VALIDATION_FILENAMES = TRAINING_FILENAMES[int(0.8*len(TRAINING_FILENAMES)) :]
TRAINING_FILENAMES = TRAINING_FILENAMES[:int(0.8*len(TRAINING_FILENAMES)) ]
y_train.value_counts()<feature_engineering>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))
x_train = x_train/255.0
test = test/255.0<train_model>
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
x_train = x_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1)
 print('x_train shape:', x_train.shape )<concatenate>
if TRAINING:
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model.summary()
y_train = keras.utils.to_categorical(y_train,num_classes=10 )<define_variables>
if TRAINING:
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB0(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model2.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model2.summary()
random_seed = 1<import_modules>
if TRAINING:
with strategy.scope() :
dnet201 = DenseNet201(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 dnet201.trainable = True
model3 = tf.keras.Sequential([
dnet201,
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model3.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model3.summary()
from sklearn.model_selection import train_test_split<import_modules>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1 )
from sklearn.model_selection import train_test_split<split>
train_dataset = get_training_dataset()
valid_dataset = get_validation_dataset()
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.1, random_state=random_seed )<import_modules>
if TRAINING:
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset 
)
 model.save("efficientnetb7.h5" )
from keras.layers import LeakyReLU<choose_model_class>
if TRAINING:
history2 = model2.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset 
)
 model2.save("efficientnetb0.h5" )
model = Sequential()
model.add(Conv2D(32,(3,3), padding='Same', activation='relu', input_shape=(28, 28, 1)))
 model.add(Dropout(0.25))
model.add(Conv2D(32,(7,7),activation='relu'))
model.add(Conv2D(128,(5,5),activation='relu'))
model.add(MaxPool2D(( 2,2)))
 model.add(Conv2D(64,(3,3), padding='Same',activation='relu'))
model.add(Dropout(0.25))
model.add(Conv2D(64,(3,3),activation='relu'))
model.add(Flatten())
 model.add(Dense(128, activation='linear'))
model.add(LeakyReLU(alpha=.001))
model.add(Dense(64, activation='linear'))
model.add(LeakyReLU(alpha=.001))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(324, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(32, activation='linear'))
model.add(LeakyReLU(alpha=.001))
model.add(Dense(10, activation='softmax'))
model.summary()<choose_model_class>
sub1 = sub.copy()
sub2 = sub.copy()
optimizer = Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)
 model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
<init_hyperparams>
if TRAINING:
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=20,
zoom_range = 0.13,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(x_train )<train_model>
if TRAINING:
pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})
 pred_df2 = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities2)})
 del sub1['target']
sub1 = sub1.merge(pred_df, on='image_name')
 sub1.to_csv('submission_efficientnetb7.csv', index=False)
 del sub2['target']
sub2 = sub2.merge(pred_df2, on='image_name')
 sub2.to_csv('submission_efficientnetb0.csv', index=False )
model.fit_generator(
 datagen.flow(x_train, y_train, batch_size=256),
steps_per_epoch=len(x_train)//256,
epochs=30,
)<compute_test_metric>
if TRAINING:
sub_es = sub1[['image_name']]
sub_es['target'] = 0.5*sub1['target'] + 0.5*sub2['target']
sub_es.to_csv('submission_blend.csv', index=False)
 sub_es.head()
score = model.evaluate(x_val, y_val, verbose=1)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )<predict_on_test>
import seaborn as sns
result = model.predict(test)
 result = np.argmax(result,axis=1)
 result = pd.Series(result,name="Label" )<save_to_csv>
sub_path = ".. /input/siimisic-submission-files"
all_files = os.listdir(sub_path)
 all_files
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),result],axis = 1)
 submission.to_csv("cnn_mnist.csv",index=False )<load_from_csv>
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "target" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
concat_sub['target_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['target_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['target_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['target_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )
print(train['SaleCondition'].unique())
 for i in range(train.shape[1]):
if train.iloc[:,i].dtypes == object:
lbl = LabelEncoder()
lbl.fit(list(train.iloc[:,i].values)+ list(test.iloc[:,i].values))
train.iloc[:,i] = lbl.transform(list(train.iloc[:,i].values))
test.iloc[:,i] = lbl.transform(list(test.iloc[:,i].values))
print(train['SaleCondition'].unique() )<prepare_x_and_y>
cutoff_lo = 0.002
cutoff_hi = 0.11
train_ID = train['Id']
test_ID = test['Id']
y_train = train['SalePrice']
X_train = train.drop(['Id','SalePrice'], axis=1)
 X_test = test.drop('Id', axis=1)
 Xmat = pd.concat([X_train, X_test] )<feature_engineering>
concat_sub['target'] = concat_sub['target_mean']
concat_sub[['image_name', 'target']].to_csv('submission_mean.csv',
index=False, float_format='%.6f' )
Xmat['TotalSF'] = Xmat['TotalBsmtSF'] + Xmat['1stFlrSF'] + Xmat['2ndFlrSF']<feature_engineering>
concat_sub['target'] = concat_sub['target_median']
concat_sub[['image_name', 'target']].to_csv('submission_median.csv',
index=False, float_format='%.6f' )
X_train = Xmat[:X_train.shape[0]]
X_test = Xmat[X_train.shape[0]:]
X_train["Interaction"] = X_train["TotalSF"] * X_train["OverallQual"]
X_test["Interaction"] = X_test["TotalSF"] * X_test["OverallQual"]
X_train.head()<import_modules>
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['target_median']))
concat_sub[['image_name', 'target']].to_csv('submission_pushout_median.csv',
index=False, float_format='%.6f' )
import lightgbm as lgb<init_hyperparams>
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['target_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['target_min'],
concat_sub['target_mean']))
concat_sub[['image_name', 'target']].to_csv('submission_minmax_mean.csv',
index=False, float_format='%.6f' )
<split><EOS> 
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['target_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['target_min'],
concat_sub['target_median']))
concat_sub[['image_name', 'target']].to_csv('submission_minmax_median.csv',
index=False, float_format='%.6f' )