Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<groupby>
!pip install efficientnet_pytorch torchtoolbox
data_all['MSZoning'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MSZoning'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Utilities'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Utilities'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior1st'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior1st'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior2nd'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior2nd'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['MasVnrType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MasVnrType'].apply(lambda x: x.fillna(x.value_counts().index[0]))
for col in ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','Alley','Fence','PoolQC','MiscFeature']:
data_all[col] = data_all[col].fillna('None')
 data_all['Electrical'] = data_all.groupby(['Neighborhood','MSSubClass' ])['Electrical'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['KitchenQual'] = data_all.groupby(['Neighborhood','MSSubClass' ])['KitchenQual'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Functional'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Functional'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['SaleType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['SaleType'].apply(lambda x: x.fillna(x.value_counts().index[0]))<filter>
%matplotlib inline
data_all.iloc[np.where(data_all.GrLivArea > 4000)]<drop_column>
warnings.simplefilter('ignore')
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(47 )
data_all = data_all.drop(data_all[data_all['Id'] == 524].index)
 data_all = data_all.drop(data_all[data_all['Id'] == 1299].index)
 train_len = train_len - 2
data_all.loc[2549,('GrLivArea')] = data_all['GrLivArea'].median()
data_all.loc[2549,('LotArea')] = data_all['LotArea'].median()<categorify>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
data_all['KitchenQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['FireplaceQu'].replace(['Ex','Gd','TA','Fa','Po', 'None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['PoolQC'].replace(['Ex','Gd','TA','Fa','None'],[5,4,3,2,1],inplace=True)
 data_all['ExterQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['ExterCond'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['BsmtQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtExposure'].replace(['Gd','Av','Mn','No','None'],[5,4,3,2,1],inplace=True)
 data_all['HeatingQC'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['MSSubClass'] = data_all['MSSubClass'].astype(str)
 data_all['YrSold'] = data_all['YrSold'].astype(str)
 data_all['MoSold'] = data_all['MoSold'].astype(str )<feature_engineering>
class MelanomaDataset(Dataset):
def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):

 self.df = df
self.imfolder = imfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __getitem__(self, index):
im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')
 x = cv2.imread(im_path)
 meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)
 if self.transforms:
x = self.transforms(x)
 if self.train:
y = self.df.iloc[index]['target']
return(x, meta), y
else:
return(x, meta)
 def __len__(self):
return len(self.df)
 class Net(nn.Module):
def __init__(self, arch, n_meta_features: int):
super(Net, self ).__init__()
self.arch = arch
if 'ResNet' in str(arch.__class__):
self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)
 if 'EfficientNet' in str(arch.__class__):
self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.ouput = nn.Linear(500 + 250, 1)
 def forward(self, inputs):

 x, meta = inputs
cnn_features = self.arch(x)
 meta_features = self.meta(meta)
 features = torch.cat(( cnn_features, meta_features), dim=1)
 output = self.ouput(features)
 return output
data_all['GarageScale'] = data_all['GarageCars'] * data_all['GarageArea']
data_all['GarageOrdinal'] = data_all['GarageQual'] + data_all['GarageCond']
data_all['AllPorch'] = data_all['OpenPorchSF'] + data_all['EnclosedPorch'] + data_all['3SsnPorch'] + data_all['ScreenPorch']
data_all['ExterOrdinal'] = data_all['ExterQual'] + data_all['ExterCond']
data_all['KitchenCombined'] = data_all['KitchenQual'] * data_all['KitchenAbvGr']
data_all['FireplaceCombined'] = data_all['FireplaceQu'] * data_all['Fireplaces']
data_all['BsmtOrdinal'] = data_all['BsmtQual'] + data_all['BsmtCond']
data_all['BsmtFinishedAll'] = data_all['BsmtFinSF1'] + data_all['BsmtFinSF2']
data_all['AllFlrSF'] = data_all['1stFlrSF'] + data_all['2ndFlrSF']
data_all['OverallCombined'] = data_all['OverallQual'] + data_all['OverallCond']
data_all['TotalFullBath'] = data_all['BsmtFullBath'] + + data_all["FullBath"]
data_all['TotalHalfBath'] = data_all["HalfBath"] + data_all['BsmtHalfBath']
data_all['TotalSF'] = data_all['AllFlrSF'] + data_all['TotalBsmtSF']
data_all['YrBltAndRemod'] = data_all["YearRemodAdd"] + data_all['YearBuilt']<categorify>
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'
cat_features = [f for f in data_all.columns if data_all.dtypes[f] == 'object']
categorical = []
for i in data_all.columns:
if i in cat_features:
categorical.append(i)
 data_cat = data_all[categorical]
data_cat = pd.get_dummies(data_cat )<count_values>
class DrawHair:

 def __init__(self, hairs:int = 4, width:tuple =(1, 2)) :
self.hairs = hairs
self.width = width
def __call__(self, img):

 if not self.hairs:
return img
width, height, _ = img.shape
for _ in range(random.randint(0, self.hairs)) :
origin =(random.randint(0, width), random.randint(0, height // 2))
end =(random.randint(0, width), random.randint(0, height))
color =(0, 0, 0)
 cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'
overfit = []
for i in data_cat.columns:
counts = data_cat[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(data_cat)* 100 > 99.95:
overfit.append(i)
 data_cat = data_cat.drop(overfit, axis=1)
 print("There are {} qualitative features with zeros > 99.95 :".format(len(overfit)) )<prepare_output>
class Microscope:

 def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img):

 if random.random() < self.p:
circle = cv2.circle(( np.ones(img.shape)* 255 ).astype(np.uint8), 
(img.shape[0]//2, img.shape[1]//2),
random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), 
(0, 0, 0),
-1)
 mask = circle - 255
img = np.multiply(img, mask)
 return img
def __repr__(self):
return f'{self.__class__.__name__}(p={self.p})'
numerical = numeric_cols.values.tolist()
numerical.remove('SalePrice')
 data_num = data_all[numerical].astype(float)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
print("There are {} numerical features with absolute Skew > 0.5 :".format(skewness.shape[0]))
skewness = pd.DataFrame({'Skew' :skewness})
 skewness.head(10 )<feature_engineering>
train_transform = transforms.Compose([
AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs'),
transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
Microscope(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ])
 test_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ] )
for i in skewness.index.tolist() :
data_num[i] = boxcox1p(data_num[i], boxcox_normmax(data_num[i] + 1))<feature_engineering>
arch = EfficientNet.from_pretrained('efficientnet-b1' )
skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
neg_skew = skewness[skewness < 0]
data_num.loc[:,('GarageOrdinal')] = data_num['GarageOrdinal']**1.1
data_num.loc[:,('GarageQual')] = data_num['GarageQual']**1.2
data_num.loc[:,('GarageCond')] = data_num['GarageCond']**1.3
data_num.loc[:,('HeatingQC')] = data_num['HeatingQC']**1.6
pos_skew = skewness[skewness > 0]
pos_skew1 = ['Fireplaces','BsmtFullBath','OpenPorchSF','FireplaceCombined','TotalHalfBath',
'WoodDeckSF','2ndFlrSF']
data_num.loc[:,pos_skew1] = np.sqrt(data_num[pos_skew1])
 data_num.loc[:,('HalfBath')] = np.cbrt(data_num['HalfBath'])
 data_num.loc[:,('MasVnrArea')] = np.log2(data_num['MasVnrArea']+1)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 1.0]
data_num = data_num.drop(skewness.index.tolist() , axis=1)
 print("There are {} numerical features with absolute Skew > 1.0 :".format(skewness.shape[0]))<prepare_x_and_y>
train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')
 test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv' )
data_num =(data_num - data_num.mean())/data_num.std()
df_all = pd.concat([data_num, data_cat], axis = 1)
 features =(data_cat.columns ).append(data_num.columns)
 df_train = df_all[:1458]
df_test = df_all[1458:]
target = 'SalePrice'
X = df_train[features]
y = data_all[:1458][target]
X_test = df_test[features]<compute_test_metric>
concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].fillna(-1)
 train_df['age_approx'] /= train_df['age_approx'].max()
test_df['age_approx'] /= test_df['age_approx'].max()
train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] = test_df['age_approx'].fillna(0)
 train_df['patient_id'] = train_df['patient_id'].fillna(0 )
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring = "neg_mean_squared_error", cv = 5))
return(rmse )<train_on_grid>
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge' )
ridge = Ridge(alpha = 14)
 cv_ridge = rmse_cv(ridge)
 ridge.fit(X , y)
 print('Ridge CV score min: ' + str(cv_ridge.min())+ ' mean: ' + str(cv_ridge.mean())
 + ' max: ' + str(cv_ridge.max()))<train_on_grid>
test = MelanomaDataset(df=test_df,
imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/',
train=False,
transforms=train_transform,
meta_features=meta_features )
lasso = Lasso(alpha =.00066)
 cv_lasso = rmse_cv(lasso)
 lasso.fit(X, y)
 print('Lasso CV score min: ' + str(cv_lasso.min())+ ' mean: ' + str(cv_lasso.mean())
 + ' max: ' + str(cv_lasso.max()))<train_on_grid>
skf = GroupKFold(n_splits=5 )
gbrt = GradientBoostingRegressor(n_estimators=1000, learning_rate=.03, max_depth=3, max_features=.04, min_samples_split=4,
min_samples_leaf=3, loss='huber', subsample=1.0, random_state=0)
 cv_gbrt = rmse_cv(gbrt)
 gbrt.fit(X, y)
 print('GradientBoosting CV score min: ' + str(cv_gbrt.min())+ ' mean: ' + str(cv_gbrt.mean())
 + ' max: ' + str(cv_gbrt.max()))<save_to_csv>
epochs = 12
es_patience = 3
TTA = 3
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 skf = KFold(n_splits=5, shuffle=True, random_state=47)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):
print('=' * 20, 'Fold', fold, '=' * 20)
 model_path = f'model_{fold}.pth'
best_val = 0
patience = es_patience
arch = EfficientNet.from_pretrained('efficientnet-b1')
 model = Net(arch=arch, n_meta_features=len(meta_features))
model = model.to(device)
 optim = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)
 criterion = nn.BCEWithLogitsLoss()
train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=train_transform,
meta_features=meta_features)
 val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=test_transform,
meta_features=meta_features)
 train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)
 val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
correct = 0
epoch_loss = 0
model.train()
for x, y in train_loader:
x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)
 x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)
 y = torch.tensor(y, device=device, dtype=torch.float32)
 optim.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
optim.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
train_acc = correct / len(train_idx)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
scheduler.step(val_roc)
 if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
oof[val_idx] = val_preds.cpu().numpy()
tta_preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 for _ in range(TTA):
for i, x_test in enumerate(test_loader):
x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)
 x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test
preds += tta_preds / TTA
preds /= skf.n_splits
def blend_models(X):
return(( ridge.predict(X)) +(gbrt.predict(X)))/2
ridge_preds = np.expm1(ridge.predict(X_test))
gbrt_preds = np.expm1(gbrt.predict(X_test))
ridge_gbrt_preds = np.expm1(blend_models(X_test))
hp_ridge_gbrt = pd.DataFrame({'Id':test.Id, 'SalePrice':ridge_gbrt_preds})
 hp_ridge_gbrt.to_csv('solution.csv', index = False )<set_options>
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )
%matplotlib inline 
<load_from_csv>
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )<prepare_x_and_y>
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission.csv', index=False )
x_train = np.array(x_train)
 test = np.array(test )<categorify>
!pip install efficientnet_pytorch torchtoolbox
y_train = to_categorical(y_train,10 )<split>
%matplotlib inline
x_train, x_val, y_train, y_val=train_test_split(x_train,y_train,test_size=0.1 )<choose_model_class>
warnings.simplefilter('ignore')
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(47 )
model = Sequential()<choose_model_class>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
model.add(Input(shape=(28, 28, 1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(1,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(MaxPooling2D(pool_size=(1,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(128,(2,2),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(128,(2,2),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))<choose_model_class>
class MelanomaDataset(Dataset):
def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):

 self.df = df
self.imfolder = imfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __getitem__(self, index):
im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')
 x = cv2.imread(im_path)
 meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)
 if self.transforms:
x = self.transforms(x)
 if self.train:
y = self.df.iloc[index]['target']
return(x, meta), y
else:
return(x, meta)
 def __len__(self):
return len(self.df)
 class Net(nn.Module):
def __init__(self, arch, n_meta_features: int):
super(Net, self ).__init__()
self.arch = arch
if 'ResNet' in str(arch.__class__):
self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)
 if 'EfficientNet' in str(arch.__class__):
self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.ouput = nn.Linear(500 + 250, 1)
 def forward(self, inputs):

 x, meta = inputs
cnn_features = self.arch(x)
 meta_features = self.meta(meta)
 features = torch.cat(( cnn_features, meta_features), dim=1)
 output = self.ouput(features)
 return output
model.compile(loss=keras.losses.categorical_crossentropy,\
optimizer = tf.keras.optimizers.Adam() ,\
metrics=['accuracy'] )<train_model>
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'
model.fit(x_train,y_train, batch_size = 128, epochs = 30, validation_data=(x_val,y_val))<predict_on_test>
class DrawHair:

 def __init__(self, hairs:int = 4, width:tuple =(1, 2)) :
self.hairs = hairs
self.width = width
def __call__(self, img):

 if not self.hairs:
return img
width, height, _ = img.shape
for _ in range(random.randint(0, self.hairs)) :
origin =(random.randint(0, width), random.randint(0, height // 2))
end =(random.randint(0, width), random.randint(0, height))
color =(0, 0, 0)
 cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'
y_pred = model.predict(x_val)
 y_pred_classes = np.argmax(y_pred,axis = 1)
 y_pred_classes
y_true = np.argmax(y_val,axis = 1)
 y_true
confusion_mtx = confusion_matrix(y_true, y_pred_classes)
 confusion_mtx<compute_test_metric>
class Microscope:

 def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img):

 if random.random() < self.p:
circle = cv2.circle(( np.ones(img.shape)* 255 ).astype(np.uint8), 
(img.shape[0]//2, img.shape[1]//2),
random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), 
(0, 0, 0),
-1)
 mask = circle - 255
img = np.multiply(img, mask)
 return img
def __repr__(self):
return f'{self.__class__.__name__}(p={self.p})'
model.evaluate(x_val,y_val,verbose=0 )<define_variables>
train_transform = transforms.Compose([
AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs'),
transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
Microscope(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ])
 test_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ] )
train_image_generator = ImageDataGenerator()<train_model>
arch = EfficientNet.from_pretrained('efficientnet-b1' )
history = model.fit_generator(train_image_generator.flow(x_train,y_train, batch_size =32),epochs = 3,validation_data =(x_val,y_val))<predict_on_test>
train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')
 test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv' )
y_pred = model.predict(x_val)
 y_pred_classes = np.argmax(y_pred,axis = 1)
 y_pred_classes<prepare_output>
concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].fillna(-1)
 train_df['age_approx'] /= train_df['age_approx'].max()
test_df['age_approx'] /= test_df['age_approx'].max()
train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] = test_df['age_approx'].fillna(0)
 train_df['patient_id'] = train_df['patient_id'].fillna(0 )
y_true = np.argmax(y_val,axis = 1)
 y_true<compute_test_metric>
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge' )
confusion_mtx = confusion_matrix(y_true, y_pred_classes)
 confusion_mtx<predict_on_test>
test = MelanomaDataset(df=test_df,
imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/',
train=False,
transforms=train_transform,
meta_features=meta_features )
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
skf = GroupKFold(n_splits=5 )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<import_modules>
epochs = 12
es_patience = 3
TTA = 3
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 skf = KFold(n_splits=5, shuffle=True, random_state=47)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):
print('=' * 20, 'Fold', fold, '=' * 20)
 model_path = f'model_{fold}.pth'
best_val = 0
patience = es_patience
arch = EfficientNet.from_pretrained('efficientnet-b1')
 model = Net(arch=arch, n_meta_features=len(meta_features))
model = model.to(device)
 optim = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)
 criterion = nn.BCEWithLogitsLoss()
train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=train_transform,
meta_features=meta_features)
 val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=test_transform,
meta_features=meta_features)
 train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)
 val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
correct = 0
epoch_loss = 0
model.train()
for x, y in train_loader:
x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)
 x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)
 y = torch.tensor(y, device=device, dtype=torch.float32)
 optim.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
optim.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
train_acc = correct / len(train_idx)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
scheduler.step(val_roc)
 if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
oof[val_idx] = val_preds.cpu().numpy()
tta_preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 for _ in range(TTA):
for i, x_test in enumerate(test_loader):
x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)
 x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test
preds += tta_preds / TTA
preds /= skf.n_splits
import tensorflow.keras as keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten
from tensorflow.keras.layers import Conv2D, MaxPool2D
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt<load_from_csv>
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )
training_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<prepare_x_and_y>
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )
<define_variables><EOS> 
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission.csv', index=False )