Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<feature_engineering>
!pip install -q efficientnet
results_df['image_id'] = results_df['image_id'].apply(lambda x: DATA_ROOT_PATH+'/'+ x+'.jpg' )<feature_engineering>
print("Tensorflow version " + tf.__version__ )
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
marking['image_id'] = marking['image_id'].apply(lambda x: TRAIN_ROOT_PATH+'/'+ x+'.jpg' )<concatenate>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('TPU Master ', tpu.master)
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print('Replicas', strategy.num_replicas_in_sync)
 DATASET1 = 'melanoma-768x768'
GCS_PATH1 = KaggleDatasets().get_gcs_path(DATASET1 )
train_data_plabel = pd.concat([results_df,marking], axis=0 )<feature_engineering>
train_details = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 train_details.isna().sum()
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = train_data_plabel[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<import_modules>
train_details['sex'] = train_details['sex'].fillna('male')
 train_details['age_approx'] = train_details['age_approx'].fillna(train_details['age_approx'].mean())
 train_details['anatom_site_general_challenge'] = train_details['anatom_site_general_challenge'].fillna('head/neck' )
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes<create_dataframe>
enc1 = LabelEncoder()
enc2 = LabelEncoder()
train_details['sex'] = enc1.fit_transform(train_details['sex'])
 train_details['anatom_site_general_challenge'] = enc2.fit_transform(train_details['anatom_site_general_challenge'])
 x_vec = train_details[['sex','age_approx','anatom_site_general_challenge']]
fold_number = 0
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True,
)<load_pretrained>
SEED = 42
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
SIZE1 = [768,768]
LR = 0.00004
EPOCHS = 12
WARMUP = 5
WEIGHT_DECAY = 0
LABEL_SMOOTHING = 0.05
TTA = 4
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<choose_model_class>
np.random.seed(SEED)
 tf.random.set_seed(SEED )
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load('.. /input/wheatgoodmodels/best-checkpoint-096epoch.bin')
 net.load_state_dict(checkpoint['model_state_dict'])
 return DetBenchTrain(net, config)
 net = get_net()<normalization>
train_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')
 test_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/test*.tfrec' )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<train_model>
train_filenames1, valid_filenames1 = train_test_split(train_filenames1, test_size = 0.15, random_state = SEED )
if len(os.listdir('.. /input/global-wheat-detection/test/')) <11:
pass
else:
run_training()<load_pretrained>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels = 3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*SIZE1, 3])
 return image
def data_augment(image, label = None, seed = SEED):
image = tf.image.rot90(image, k = np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed = SEED)
 image = tf.image.random_flip_up_down(image, seed = SEED)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)}
example = tf.io.parse_single_example(example, LFormat)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)}
example = tf.io.parse_single_example(example, UFormat)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset =(tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO ).with_options(ignore_order ).map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
weights = f'plabel_model/last-checkpoint1.bin'
if not os.path.exists(weights):
weights = '.. /input/wheatgoodmodels/best-checkpoint-096epoch.bin'
net = load_net(weights )<categorify>
train1 =(load_dataset(train_filenames1 ).map(data_augment, num_parallel_calls = AUTO ).shuffle(SEED ).batch(BATCH_SIZE, drop_remainder = True ).repeat().prefetch(AUTO))
valid_dataset1 =(load_dataset(valid_filenames1, labeled=True ).batch(BATCH_SIZE ).repeat().prefetch(AUTO))
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
with strategy.scope() :
model = tf.keras.Sequential([efn.EfficientNetB7(input_shape=(*SIZE1,3), weights='imagenet', include_top=False, pooling = 'avg'),
Dense(1, activation = 'sigmoid')])
 model.compile(optimizer='adam',loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
metrics=[tf.keras.metrics.AUC(name='auc')] )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<set_options>
validationsize1 = count_data_items(valid_filenames1)
 print(validationsize1 )
plt.style.use('classic')
 %matplotlib inline
sb.set()
warnings.filterwarnings('ignore' )<load_from_csv>
STEPS_PER_EPOCH =(count_data_items(train_filenames1)) // BATCH_SIZE
model.fit(train1, epochs=EPOCHS, callbacks=[lr_schedule],steps_per_epoch=STEPS_PER_EPOCH,validation_data=valid_dataset1,validation_steps=validationsize1//BATCH_SIZE )
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train_len = len(train)
 data_all = train.append(test, sort=False, ignore_index=True)
 features = data_all.shape[1]
idUnique = len(set(data_all.Id))
idTotal = data_all.shape[0]
idDupli = idTotal - idUnique
print("There are " + str(idDupli)+ " duplicate IDs for " + str(idTotal)+ " total entries")
 print("There are " + str(features)+ " variables for " + str(idTotal)+ " total entries" )<define_variables>
num_test_images = count_data_items(test_filenames1)
 submission_df1 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames1, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df1.copy()
del temp['target']
submission_df1['target'] += temp.merge(pred_df,on="image_name")['target']/TTA
quantity = [f for f in data_all.columns if data_all.dtypes[f] != 'object']
quantity.remove('SalePrice')
 quantity.remove('Id')
 quality = [f for f in data_all.columns if data_all.dtypes[f] == 'object']<filter>
submission_df1.to_csv('efficientnetb7_784.csv', index = False )
missing_quant =(data_all[quantity].isnull().sum() /data_all[quantity].isnull().count() ).sort_values(ascending=False)
 missing_quant = missing_quant[missing_quant > 0] * 100
print("There are {} quantitative features with missing values :".format(missing_quant.shape[0]))
missing_quant = pd.DataFrame({'Percent' :missing_quant})
 missing_quant.head()<filter>
DATASET2 = 'melanoma-512x512'
GCS_PATH2 = KaggleDatasets().get_gcs_path(DATASET2)
 SIZE2 = [512,512]
train_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 test_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/test*.tfrec')
 train_filenames2, valid_filenames2 = train_test_split(train_filenames2, test_size = 0.15, random_state = SEED)
 def decode_image(image):
image = tf.image.decode_jpeg(image, channels = 3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*SIZE2, 3])
 return image
def data_augment(image, label = None, seed = SEED):
image = tf.image.rot90(image, k = np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed = SEED)
 image = tf.image.random_flip_up_down(image, seed = SEED)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)}
example = tf.io.parse_single_example(example, LFormat)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)}
example = tf.io.parse_single_example(example, UFormat)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset =(tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO ).with_options(ignore_order ).map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 train2 =(load_dataset(train_filenames2 ).map(data_augment, num_parallel_calls = AUTO ).shuffle(SEED ).batch(BATCH_SIZE, drop_remainder = True ).repeat().prefetch(AUTO))
valid_dataset2 =(load_dataset(valid_filenames2, labeled=True ).batch(BATCH_SIZE ).repeat().prefetch(AUTO))
np.where(pd.isnull(data_all.GarageArea))<data_type_conversions>
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB7(input_shape=(*SIZE2, 3),weights='imagenet',pooling='avg',include_top=False),
Dense(1, activation='sigmoid')
 ])
 model2.compile(optimizer='adam',loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
metrics=[tf.keras.metrics.AUC(name='auc')] )
data_all['LotFrontage'] = data_all.groupby(['Neighborhood'])['LotFrontage'].apply(lambda x: x.fillna(x.median()))
data_all.at[2576, 'GarageYrBlt'] = 0
data_all['GarageYrBlt'] = data_all.groupby(['Neighborhood'])['GarageYrBlt'].apply(lambda x: x.fillna(x.median()))
for col in ['MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageCars',
'GarageArea']:
data_all[col] = data_all[col].fillna(0 )<sort_values>
validationsize2 = count_data_items(valid_filenames2)
 print(validationsize2 )
missing_qual =(data_all[quality].isnull().sum() /data_all[quality].isnull().count() ).sort_values(ascending=False)
 missing_qual = missing_qual[missing_qual > 0] * 100
print("There are {} qualitative features with missing values :".format(missing_qual.shape[0]))
missing_qual = pd.DataFrame({'Percent' :missing_qual})
 missing_qual.head(10 )<groupby>
STEPS_PER_EPOCH =(count_data_items(train_filenames2)) // BATCH_SIZE
model2.fit(train2, epochs=EPOCHS, callbacks=[lr_schedule],steps_per_epoch=STEPS_PER_EPOCH,validation_data=valid_dataset2,validation_steps=validationsize2//BATCH_SIZE )
data_all['MSZoning'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MSZoning'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Utilities'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Utilities'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior1st'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior1st'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior2nd'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior2nd'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['MasVnrType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MasVnrType'].apply(lambda x: x.fillna(x.value_counts().index[0]))
for col in ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','Alley','Fence','PoolQC','MiscFeature']:
data_all[col] = data_all[col].fillna('None')
 data_all['Electrical'] = data_all.groupby(['Neighborhood','MSSubClass' ])['Electrical'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['KitchenQual'] = data_all.groupby(['Neighborhood','MSSubClass' ])['KitchenQual'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Functional'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Functional'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['SaleType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['SaleType'].apply(lambda x: x.fillna(x.value_counts().index[0]))<filter>
num_test_images = count_data_items(test_filenames2)
 submission_df2 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames2, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model2.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df2.copy()
del temp['target']
submission_df2['target'] += temp.merge(pred_df,on="image_name")['target']/TTA
<drop_column><EOS> 
submission_df2.to_csv('efficientnetb7_512.csv', index=False )