Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<categorify>
fold_number = 1
seed = 66
debug = False
tta = 2 if debug else 20
batch_size = {
'tpu': 10,
'gpu': 22,
'cpu': 4,
}
arch = 'efficientnet-b5'
resolution = 456
input_res = 512
lr = 8e-6
weight_decay = 2e-5
pos_weight = 3.2
label_smoothing = 0.03
max_epochs = 7
data_all['KitchenQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['FireplaceQu'].replace(['Ex','Gd','TA','Fa','Po', 'None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['PoolQC'].replace(['Ex','Gd','TA','Fa','None'],[5,4,3,2,1],inplace=True)
 data_all['ExterQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['ExterCond'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['BsmtQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtExposure'].replace(['Gd','Av','Mn','No','None'],[5,4,3,2,1],inplace=True)
 data_all['HeatingQC'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['MSSubClass'] = data_all['MSSubClass'].astype(str)
 data_all['YrSold'] = data_all['YrSold'].astype(str)
 data_all['MoSold'] = data_all['MoSold'].astype(str )<feature_engineering>
%reload_ext autoreload
%autoreload 2
if 'TPU_NAME' in os.environ.keys() :
try:
except:
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev
else:
!pip install --pre torch==1.7.0.dev20200701+cu101 torchvision==0.8.0.dev20200701+cu101 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html
!pip install -U pip albumentations==0.4.5 PyYAML pytorch-lightning==0.8.5 efficientnet_pytorch
data_all['GarageScale'] = data_all['GarageCars'] * data_all['GarageArea']
data_all['GarageOrdinal'] = data_all['GarageQual'] + data_all['GarageCond']
data_all['AllPorch'] = data_all['OpenPorchSF'] + data_all['EnclosedPorch'] + data_all['3SsnPorch'] + data_all['ScreenPorch']
data_all['ExterOrdinal'] = data_all['ExterQual'] + data_all['ExterCond']
data_all['KitchenCombined'] = data_all['KitchenQual'] * data_all['KitchenAbvGr']
data_all['FireplaceCombined'] = data_all['FireplaceQu'] * data_all['Fireplaces']
data_all['BsmtOrdinal'] = data_all['BsmtQual'] + data_all['BsmtCond']
data_all['BsmtFinishedAll'] = data_all['BsmtFinSF1'] + data_all['BsmtFinSF2']
data_all['AllFlrSF'] = data_all['1stFlrSF'] + data_all['2ndFlrSF']
data_all['OverallCombined'] = data_all['OverallQual'] + data_all['OverallCond']
data_all['TotalFullBath'] = data_all['BsmtFullBath'] + + data_all["FullBath"]
data_all['TotalHalfBath'] = data_all["HalfBath"] + data_all['BsmtHalfBath']
data_all['TotalSF'] = data_all['AllFlrSF'] + data_all['TotalBsmtSF']
data_all['YrBltAndRemod'] = data_all["YearRemodAdd"] + data_all['YearBuilt']<categorify>
num_workers = 2
gpus = 1 if torch.cuda.is_available() else None
try:
tpu_cores = 8
except:
tpu_cores = None
if isinstance(batch_size, dict):
if tpu_cores:
batch_size = batch_size['tpu']
lr *= tpu_cores
num_workers = 1
elif gpus:
batch_size = batch_size['gpu']
if 'K80' in torch.cuda.get_device_name() :
batch_size = batch_size//3
elif 'T4' in torch.cuda.get_device_name() :
batch_size = int(batch_size * 0.66)
 else:
batch_size = batch_size['cpu']
lr *= batch_size
dict(
 num_workers=num_workers,
tpu_cores=tpu_cores,
gpus=gpus,
batch_size=batch_size,
lr=lr,
)
cat_features = [f for f in data_all.columns if data_all.dtypes[f] == 'object']
categorical = []
for i in data_all.columns:
if i in cat_features:
categorical.append(i)
 data_cat = data_all[categorical]
data_cat = pd.get_dummies(data_cat )<count_values>
if gpus and not hasattr(torch.cuda, "amp"):
try:
except:
!git clone https://github.com/NVIDIA/apex nv_apex
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext"./nv_apex
overfit = []
for i in data_cat.columns:
counts = data_cat[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(data_cat)* 100 > 99.95:
overfit.append(i)
 data_cat = data_cat.drop(overfit, axis=1)
 print("There are {} qualitative features with zeros > 99.95 :".format(len(overfit)) )<prepare_output>
warnings.filterwarnings("ignore")
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(seed*6 + fold_number)
 torch.__version__
numerical = numeric_cols.values.tolist()
numerical.remove('SalePrice')
 data_num = data_all[numerical].astype(float)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
print("There are {} numerical features with absolute Skew > 0.5 :".format(skewness.shape[0]))
skewness = pd.DataFrame({'Skew' :skewness})
 skewness.head(10 )<feature_engineering>
class ImageDataset(Dataset):
def __init__(self, path, image_ids, labels=None, transforms=None):
super().__init__()
self.path = path
self.image_ids = image_ids
self.labels = labels
self.transforms = transforms
def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.path}/{image_id}.jpg', cv2.IMREAD_COLOR)
 if self.transforms:
sample = self.transforms(image=image)
 image = sample['image']
label = self.labels[idx] if self.labels is not None else 0.5
return image, label
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_labels(self):
return list(self.labels )
for i in skewness.index.tolist() :
data_num[i] = boxcox1p(data_num[i], boxcox_normmax(data_num[i] + 1))<feature_engineering>
def get_train_transforms() :
return A.Compose([
A.JpegCompression(p=0.5),
A.Rotate(limit=80, p=1.0),
A.OneOf([
A.OpticalDistortion() ,
A.GridDistortion() ,
A.IAAPiecewiseAffine() ,
]),
A.RandomSizedCrop(min_max_height=(int(resolution*0.7), input_res),
height=resolution, width=resolution, p=1.0),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.GaussianBlur(p=0.3),
A.OneOf([
A.RandomBrightnessContrast() ,
A.HueSaturationValue() ,
]),
A.Cutout(num_holes=8, max_h_size=resolution//8, max_w_size=resolution//8, fill_value=0, p=0.3),
A.Normalize() ,
ToTensorV2() ,
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.CenterCrop(height=resolution, width=resolution, p=1.0),
A.Normalize() ,
ToTensorV2() ,
], p=1.0)
 def get_tta_transforms() :
return A.Compose([
A.JpegCompression(p=0.5),
A.RandomSizedCrop(min_max_height=(int(resolution*0.9), int(resolution*1.1)) ,
height=resolution, width=resolution, p=1.0),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Transpose(p=0.5),
A.Normalize() ,
ToTensorV2() ,
], p=1.0 )
skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
neg_skew = skewness[skewness < 0]
data_num.loc[:,('GarageOrdinal')] = data_num['GarageOrdinal']**1.1
data_num.loc[:,('GarageQual')] = data_num['GarageQual']**1.2
data_num.loc[:,('GarageCond')] = data_num['GarageCond']**1.3
data_num.loc[:,('HeatingQC')] = data_num['HeatingQC']**1.6
pos_skew = skewness[skewness > 0]
pos_skew1 = ['Fireplaces','BsmtFullBath','OpenPorchSF','FireplaceCombined','TotalHalfBath',
'WoodDeckSF','2ndFlrSF']
data_num.loc[:,pos_skew1] = np.sqrt(data_num[pos_skew1])
 data_num.loc[:,('HalfBath')] = np.cbrt(data_num['HalfBath'])
 data_num.loc[:,('MasVnrArea')] = np.log2(data_num['MasVnrArea']+1)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 1.0]
data_num = data_num.drop(skewness.index.tolist() , axis=1)
 print("There are {} numerical features with absolute Skew > 1.0 :".format(skewness.shape[0]))<prepare_x_and_y>
DATA_PATH = '.. /input/melanoma-merged-external-data-512x512-jpeg'
TRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'
TEST_ROOT_PATH = f'{DATA_PATH}/512x512-test/512x512-test'
df_folds = pd.read_csv(f'{DATA_PATH}/folds.csv', index_col='image_id',
usecols=['image_id', 'fold', 'target'], dtype={'fold': np.byte, 'target': np.byte})
 _ = df_folds.groupby('fold' ).target.hist(alpha=0.4)
 df_folds.groupby('fold' ).target.mean().to_frame('ratio' ).T
data_num =(data_num - data_num.mean())/data_num.std()
df_all = pd.concat([data_num, data_cat], axis = 1)
 features =(data_cat.columns ).append(data_num.columns)
 df_train = df_all[:1458]
df_test = df_all[1458:]
target = 'SalePrice'
X = df_train[features]
y = data_all[:1458][target]
X_test = df_test[features]<compute_test_metric>
df_test = pd.read_csv(f'.. /input/siim-isic-melanoma-classification/test.csv', index_col='image_name')
 if debug:
df_folds = df_folds.sample(batch_size * 80)
 df_folds = df_folds.sample(frac=1.0, random_state=seed*6+fold_number )
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring = "neg_mean_squared_error", cv = 5))
return(rmse )<train_on_grid>
ds_train = ImageDataset(
 path=TRAIN_ROOT_PATH,
image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
labels=df_folds[df_folds['fold'] != fold_number].target.values,
transforms=get_train_transforms() , 
)
 ds_val = ImageDataset(
 path=TRAIN_ROOT_PATH,
image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
labels=df_folds[df_folds['fold'] == fold_number].target.values,
transforms=get_valid_transforms() , 
)
 ds_test = ImageDataset(
 path=TEST_ROOT_PATH,
image_ids=df_test.index.values,
transforms=get_tta_transforms() , 
)
 del df_folds
len(ds_train), len(ds_val), len(ds_test )
ridge = Ridge(alpha = 14)
 cv_ridge = rmse_cv(ridge)
 ridge.fit(X , y)
 print('Ridge CV score min: ' + str(cv_ridge.min())+ ' mean: ' + str(cv_ridge.mean())
 + ' max: ' + str(cv_ridge.max()))<train_on_grid>
class Model(pl.LightningModule):
def __init__(self, *args, **kwargs):
super().__init__()
self.net = EfficientNet.from_pretrained(arch, advprop=True)
 self.net._fc = nn.Linear(in_features=self.net._fc.in_features, out_features=1, bias=True)
 def forward(self, x):
return self.net(x)
 def configure_optimizers(self):
optimizer = torch.optim.AdamW(self.parameters() , lr=lr, weight_decay=weight_decay)
 scheduler = torch.optim.lr_scheduler.OneCycleLR(
 max_lr=lr,
epochs=max_epochs,
optimizer=optimizer,
steps_per_epoch=int(len(ds_train)/ batch_size),
pct_start=0.1,
div_factor=10,
final_div_factor=100,
base_momentum=0.90,
max_momentum=0.95, 
)
 return [optimizer], [scheduler]
def step(self, batch):
x, y = batch
y_hat = self(x ).flatten()
y_smo = y.float() *(1 - label_smoothing)+ 0.5 * label_smoothing
loss = F.binary_cross_entropy_with_logits(y_hat, y_smo.type_as(y_hat),
pos_weight=torch.tensor(pos_weight))
return loss, y, y_hat.sigmoid()
def training_step(self, batch, batch_nb):
loss, y, y_hat = self.step(batch)
 acc =(y_hat.round() == y ).float().mean().item()
tensorboard_logs = {'train_loss': loss, 'acc': acc}
return {'loss': loss, 'acc': acc, 'log': tensorboard_logs}
def validation_step(self, batch, batch_nb):
loss, y, y_hat = self.step(batch)
 return {'val_loss': loss,
'y': y.detach() , 'y_hat': y_hat.detach() }
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs] ).mean()
y = torch.cat([x['y'] for x in outputs])
 y_hat = torch.cat([x['y_hat'] for x in outputs])
 auc = AUROC()(pred=y_hat, target=y)if y.float().mean() > 0 else 0.5
acc =(y_hat.round() == y ).float().mean().item()
print(f"Epoch {self.current_epoch} acc:{acc} auc:{auc}")
 tensorboard_logs = {'val_loss': avg_loss, 'val_auc': auc, 'val_acc': acc}
return {'avg_val_loss': avg_loss,
'val_auc': auc, 'val_acc': acc,
'log': tensorboard_logs}
def test_step(self, batch, batch_nb):
x, _ = batch
y_hat = self(x ).flatten().sigmoid()
return {'y_hat': y_hat}
def test_epoch_end(self, outputs):
y_hat = torch.cat([x['y_hat'] for x in outputs])
 df_test['target'] = y_hat.tolist()
N = len(glob('submission*.csv'))
df_test.target.to_csv(f'submission{N}.csv')
 return {'tta': N}
def train_dataloader(self):
return DataLoader(ds_train, batch_size=batch_size, num_workers=num_workers,
drop_last=True, shuffle=True, pin_memory=True)
 def val_dataloader(self):
return DataLoader(ds_val, batch_size=batch_size, num_workers=num_workers,
drop_last=False, shuffle=False, pin_memory=True)
 def test_dataloader(self):
return DataLoader(ds_test, batch_size=batch_size, num_workers=num_workers,
drop_last=False, shuffle=False, pin_memory=False)
 model = Model()
lasso = Lasso(alpha =.00066)
 cv_lasso = rmse_cv(lasso)
 lasso.fit(X, y)
 print('Lasso CV score min: ' + str(cv_lasso.min())+ ' mean: ' + str(cv_lasso.mean())
 + ' max: ' + str(cv_lasso.max()))<train_on_grid>
del batch; del targets
gbrt = GradientBoostingRegressor(n_estimators=1000, learning_rate=.03, max_depth=3, max_features=.04, min_samples_split=4,
min_samples_leaf=3, loss='huber', subsample=1.0, random_state=0)
 cv_gbrt = rmse_cv(gbrt)
 gbrt.fit(X, y)
 print('GradientBoosting CV score min: ' + str(cv_gbrt.min())+ ' mean: ' + str(cv_gbrt.mean())
 + ' max: ' + str(cv_gbrt.max()))<save_to_csv>
def blend_models(X):
return(( ridge.predict(X)) +(gbrt.predict(X)))/2
ridge_preds = np.expm1(ridge.predict(X_test))
gbrt_preds = np.expm1(gbrt.predict(X_test))
ridge_gbrt_preds = np.expm1(blend_models(X_test))
hp_ridge_gbrt = pd.DataFrame({'Id':test.Id, 'SalePrice':ridge_gbrt_preds})
 hp_ridge_gbrt.to_csv('solution.csv', index = False )<set_options>
checkpoint_callback = pl.callbacks.ModelCheckpoint("{epoch:02d}_{val_auc:.4f}",
save_top_k=1, monitor='val_auc', mode='max')
 trainer = pl.Trainer(
 tpu_cores=tpu_cores,
gpus=gpus,
precision=16 if gpus else 32,
max_epochs=max_epochs,
num_sanity_val_steps=1 if debug else 0,
checkpoint_callback=checkpoint_callback,
)
plt.style.use('classic')
 %matplotlib inline
sb.set()
warnings.filterwarnings('ignore' )<load_from_csv>
torch.cuda.empty_cache() ; gc.collect()
torch.cuda.empty_cache() ; gc.collect()
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train_len = len(train)
 data_all = train.append(test, sort=False, ignore_index=True)
 features = data_all.shape[1]
idUnique = len(set(data_all.Id))
idTotal = data_all.shape[0]
idDupli = idTotal - idUnique
print("There are " + str(idDupli)+ " duplicate IDs for " + str(idTotal)+ " total entries")
 print("There are " + str(features)+ " variables for " + str(idTotal)+ " total entries" )<define_variables>
trainer.fit(model )
quantity = [f for f in data_all.columns if data_all.dtypes[f] != 'object']
quantity.remove('SalePrice')
 quantity.remove('Id')
 quality = [f for f in data_all.columns if data_all.dtypes[f] == 'object']<filter>
submission = df_test[['target']]
submission.target = 0.0
for sub in glob('submission*.csv'):
submission.target += pd.read_csv(sub, index_col='image_name' ).target
submission.target -= submission.target.min()
submission.target /= submission.target.max()
submission.to_csv(f'submission_fold{fold_number}.csv')
 submission.hist(bins=100, log=True, alpha=0.6)
 submission.target.describe()
missing_quant =(data_all[quantity].isnull().sum() /data_all[quantity].isnull().count() ).sort_values(ascending=False)
 missing_quant = missing_quant[missing_quant > 0] * 100
print("There are {} quantitative features with missing values :".format(missing_quant.shape[0]))
missing_quant = pd.DataFrame({'Percent' :missing_quant})
 missing_quant.head()<filter>
folds_path = '.. /input/melanoma-neat-pytorch-lightning'
!cp {folds_path}/*_fold*.csv.
 !cp {folds_path}/*.ckpt .
np.where(pd.isnull(data_all.GarageArea))<data_type_conversions>
folds_sub = pd.read_csv(f'{folds_path}/submission.csv', index_col='image_name')
 submission.target += folds_sub.target *(fold_number + 4)
 submission.target /=(fold_number + 5)
 submission.to_csv('submission.csv')
 submission.hist(bins=100, log=True, alpha=0.6)
 submission.target.describe()
<sort_values><EOS> 
if not debug and gpus:
!rm nv_apex -rf
!ls -sh