Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<train_model>
!pip install efficientnet_pytorch torchtoolbox
y_train_trf = np.log1p(y_train)
 y_test_trf = np.log1p(y_test)
 xgb_pipe_trf = Pipeline(grid.best_estimator_.steps[:-1])
 xgb_pipe_trf.fit(X_train, y_train_trf)
 X_train_trf = xgb_pipe_trf.transform(X_train)
 X_test_trf = xgb_pipe_trf.transform(X_test )<train_model>
%matplotlib inline
xgb_fit_params = {
"early_stopping_rounds": 10,
"eval_set" : [(X_test_trf, y_test_trf)],
"verbose": False
}
xgb_model = xgb.XGBRegressor(objective="reg:squaredlogerror" )<train_on_grid>
warnings.simplefilter('ignore')
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(47 )
one_to_left = st.beta(10, 1)
 from_zero_positive = st.expon(0, 50)
 xgb_params = {
"n_estimators": st.randint(3, 2000),
"max_depth": st.randint(3, 60),
"learning_rate": st.uniform(0.05, 0.4),
"colsample_bytree": one_to_left,
"subsample": one_to_left,
"gamma": st.uniform(0, 10),
"reg_alpha": from_zero_positive,
"min_child_weight": from_zero_positive,
}
xgb_grid = RandomizedSearchCV(
 xgb_model,
param_distributions=xgb_params,
n_iter=20,
cv=5,
scoring="neg_mean_squared_log_error", 
)
 xgb_grid.fit(X_train_trf, y_train_trf, **xgb_fit_params )<find_best_params>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
xgb_grid.best_params_<train_model>
basepath = ".. /input/jpeg-melanoma-256x256/"
train_csv = basepath + "train.csv"
test_csv = basepath + "test.csv"
train_img_path = '.. /input/jpeg-melanoma-256x256/train/'
test_img_path = '.. /input/jpeg-melanoma-256x256/test/'
print(train_csv )
xgb_pipe = Pipeline([*xgb_pipe_trf.steps,("model", transform_regressor(xgb_grid.best_estimator_)) ])
 xgb_pipe.fit(X_train, y_train, **prepend_keys(xgb_fit_params, "model__"))<compute_test_metric>
train_df = pd.read_csv(train_csv)
 test_df = pd.read_csv(test_csv)
 display(train_df.head())
 print("Train shape: ", train_df.shape)
 print("Test shape: ",test_df.shape )
md(f"Model score on the validation set: `{rmsle(y_test, xgb_pipe.predict(X_test)) :.3f}`")
 md(f"Model score on the train set: `{rmsle(y_train, xgb_pipe.predict(X_train)) :.3f}`" )<create_dataframe>
location = 'anatom_site_general_challenge'
concat = pd.concat([train_df[location], test_df[location]], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]].reset_index(drop=True)], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].fillna(-1)
 train_df['age_approx'] /= train_df['age_approx'].max()
train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] /= test_df['age_approx'].max()
test_df['age_approx'] = test_df['age_approx'].fillna(0)
 print("Train shape: ", train_df.shape)
 print("Test shape: ", test_df.shape )
submission = pd.DataFrame({"SalePrice": grid.predict(test)}, index=test.index)
 submission<save_to_csv>
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge')
 meta_features
submission.to_csv("submission.csv" )<import_modules>
class SIIMDataset(Dataset):
def __init__(self, df: pd.DataFrame, imgfolder: str, train: bool = True, transforms = None, meta_features = None):
self.df = df
self.imgfolder = imgfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
im_path = os.path.join(self.imgfolder, self.df.iloc[idx]['image_name'] + '.jpg')
 img = cv2.imread(im_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 meta = np.array(self.df.iloc[idx][self.meta_features].values, dtype=np.float32)
 meta = torch.from_numpy(meta)
 if self.transforms:
img = self.transforms(img)
 if self.train:
target = torch.tensor(self.df.loc[idx, 'target'], dtype=torch.float)
 return(img, meta), target
else:
return(img, meta )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
ones = len(train_df.query('target == 1'))
zeros = len(train_df.query('target == 0'))
weightage_fn = {0: 1./zeros, 1: 1./ones}
print(weightage_fn )
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv' ).fillna(' ')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' ).fillna(' ')
 keep_train_index=train['text'].drop_duplicates().index
train=train.iloc[keep_train_index,:]
train_text = train["text"]
test_text=test['text']
all_text = pd.concat([train_text, test_text] )<categorify>
def get_sampler(df, idx):
targets = df['target'][idx].values
weights = [weightage_fn[x] for x in targets]
sampler = torch.utils.data.WeightedRandomSampler(weights=weights, num_samples=len(weights), replacement=True)
 return sampler
def replace_contractions(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet<string_transform>
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 10, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'
custom_stop_list = []
stopword_set = stopwords.words('english')+custom_stop_list+['url']
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 remove_emoji("Omg another Earthquake 😔😔")
 def preProcess(iter):
regex_ws=re.compile("\s+")
 ret=regex_ws.sub(" ",iter)
 ret=ret.replace("&amp;","&" ).replace("&lt;","<" ).replace("&gt;",">")
 regexp="(https?:\/\/(?:www\.|(?!www)|(?:xmlns\.))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www)) [a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})"
ret=re.sub(regexp,"url",ret)
 regexp='@[A-z0-9_]+'
ret=re.sub(regexp,"@twitterhandle",ret)
 ret=remove_emoji(ret)
 ret1=re.split("[,_, \<>!\?\.:
"=*/]+",ret)
 ret2=[word for word in ret1 if word not in stopword_set]
ret2=" ".join(ret2)
 ret2=re.sub(r"(\s\d+)"," ",ret2)
 return ret2
te=preProcess("I'll make the day a fantastic 1!!")
 print(te )<feature_engineering>
dataset = SIIMDataset(df=train_df, imgfolder=train_img_path, train=True, transforms=train_transforms, meta_features=meta_features)
 loader = DataLoader(dataset=dataset, batch_size=16, shuffle=True)
 data = iter(loader)
 images = data.next()
show_images(torchvision.utils.make_grid(images[0][0]))
del dataset, loader
print("TFIDF")
 word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 1),
norm='l2',
min_df=0,
smooth_idf=False,
preprocessor=preProcess,
max_features=15000)
 word_vectorizer.fit(all_text)
 train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text)
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 6),
norm='l2',
min_df=0,
smooth_idf=False,preprocessor=preProcess,
max_features=30000)
 char_vectorizer.fit(all_text)
 train_char_features = char_vectorizer.transform(train_text)
 test_char_features = char_vectorizer.transform(test_text)
 enc = OneHotEncoder(handle_unknown='ignore')
 train_keyword_features = enc.fit_transform(train[['keyword']].to_numpy().reshape(-1,1))
test_keyword_features = enc.transform(test[['keyword']].to_numpy().reshape(-1,1))
train_features = hstack([train_char_features, train_word_features] ).tocsr()
del train_char_features,train_word_features
test_features = hstack([test_char_features, test_word_features] ).tocsr()
del test_char_features,test_word_features
print(train_features.shape)
 print(test_features.shape )<compute_test_metric>
class SIIMNet(nn.Module):
def __init__(self, base1, base2, n_meta_features: int):
super(SIIMNet, self ).__init__()
self.base1 = base1
self.base1._fc = nn.Linear(in_features=1536, out_features=500, bias=True)
 self.base2 = base2
self.base2._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.fc1 = nn.Linear(1000, 500)
 self.bn1 = nn.BatchNorm1d(500)
 self.meta_net = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.out = nn.Linear(500+250, 1)
 def forward(self, inputs):
img, meta = inputs
cnn1_ = self.base1(img)
 cnn2_ = self.base2(img)
 cnn_ = torch.cat(( cnn1_, cnn2_), 1)
 cnn_ = self.fc1(cnn_)
 cnn_ = self.bn1(cnn_)
 meta_ = self.meta_net(meta)
 features = torch.cat(( cnn_, meta_), dim=1)
 output = self.out(features)
 return output
def f1_metric(preds,dtrain):
ytrue=dtrain.get_label()
return 'f1_score', 1-metrics.f1_score(preds.round().astype(np.int), ytrue, average='macro' )<train_model>
epochs = 10
model_path = 'model.pth'
es_patience = 3
TTA = 3
skf = StratifiedKFold(n_splits=5 )
def train_model(train_features, train_target,test_features,n_estimator):
d_test = xgb.DMatrix(test_features)
 gc.collect()
print("Modeling")
 cv_scores = []
xgb_preds = []
submission = pd.DataFrame.from_dict({'id': test['id']})
 xgb_params = {'eta': 0.1,
'max_depth': 7,
'subsample': 0.7,
'colsample_bytree': 0.75,
'n_estimators':n_estimator,
'objective': 'binary:logistic',
'metric': 'f1_score',
'eval_metric':'error',
'gamma':0,
'seed': 314159265,
'verbose':20,
'min_child_weight':1
}
cv=StratifiedKFold(n_splits=4)
 for i,(train_index, valid_index)in enumerate(cv.split(train_features, train_target)) :
X_train, X_valid=train_features[train_index],train_features[valid_index]
y_train, y_valid = train_target.iloc[train_index],train_target.iloc[valid_index]
d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 watchlist = [(d_valid, 'valid')]
model = xgb.train(xgb_params, d_train, 200, watchlist, verbose_eval=False,early_stopping_rounds=100, feval=f1_metric)
 print(model.attributes())
 print(model.attributes() ['best_msg'])
 cv_scores.append(float(model.attributes() ['best_score']))
submission[i] = model.predict(d_test)
 gc.collect()
print('Total CV score is {}'.format(np.mean(cv_scores)))

print(model.attributes())
 print(model.attributes() ['best_msg'])
 cv_scores.append(float(model.attributes() ['best_score']))
submission = model.predict(d_test)
 return submission 
<count_values>
def train_fn(model, train_loader, opt, criterion):
batch = 1
epoch_loss = 0
correct = 0
for x, y in train_loader:
if(batch % 40 == 0):
print("=", end="")
 x[0] = x[0].to(device)
 x[1] = x[1].to(device)
 y = y.to(device)
 opt.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
opt.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
batch += 1
return correct, epoch_loss
train_target = train['target']
print(train_target.shape)
 train_target.value_counts()<train_model>
def validate_fn(model, val_loader, val_idx):
batch = 1
val_preds = torch.zeros(( len(val_idx), 1), device=device, dtype=torch.float32)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
if(batch % 20 == 0):
print("=", end="")
 x_val[0] = x_val[0].to(device)
 x_val[1] = x_val[1].to(device)
 y_val = y_val.to(device)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size : j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
batch += 1
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 return val_acc, val_roc
submission_tfidf=train_model(train_features, train_target,test_features,n_estimator=97)
<train_model>
def engine() :
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test_df), 1), dtype=torch.float32, device=device)
 test_ds = SIIMDataset(
 df = test_df,
imgfolder = test_img_path,
train = False,
transforms = train_transforms,
meta_features = meta_features 
)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), start=1):
print("
")
 print("="*15, "FOLD ", fold, "="*15)
 best_val = None
patience = es_patience
base1 = EfficientNet.from_pretrained('efficientnet-b3')
 base2 = EfficientNet.from_pretrained('efficientnet-b1')
 model = SIIMNet(base1=base1, base2=base2, n_meta_features=len(meta_features))
model.to(device)
 opt = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=opt, mode='max', patience=1, factor=0.2, verbose=True)
 criterion = nn.BCEWithLogitsLoss()
train_ds = SIIMDataset(
 df = train_df.iloc[train_idx].reset_index(drop=True),
imgfolder = train_img_path,
train = True,
transforms = train_transforms,
meta_features = meta_features 
)
 sampler = get_sampler(train_df, train_idx)
 train_loader = DataLoader(dataset=train_ds, batch_size=32, sampler=sampler, num_workers=2)
 val_ds = SIIMDataset(
 df = train_df.iloc[val_idx].reset_index(drop=True),
imgfolder = train_img_path,
train = True,
transforms = test_transforms,
meta_features = meta_features 
)
 val_loader = DataLoader(dataset=val_ds, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test_ds, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
print("Training:")
 model.train()
correct, epoch_loss = train_fn(model, train_loader, opt, criterion)
 train_acc = correct / len(train_idx)
 print("
Validating:")
 model.eval()
val_acc, val_roc = validate_fn(model, val_loader, val_idx)
 scheduler.step(val_roc)
 print('
Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
if not best_val:
best_val = val_roc
torch.save(model, model_path)
 continue
if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 val_preds = torch.zeros(( len(val_idx), 1), device=device, dtype=torch.float32)
 batch = 1
print("Validating model for FOLD {}:".format(fold))
model.eval()
with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
if(batch % 20 == 0):
print("=", end="")
 x_val[0] = x_val[0].to(device)
 x_val[1] = x_val[1].to(device)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size : j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
batch += 1
oof[val_idx] = val_preds.cpu().numpy()
batch = 1
print("
Testing:")
 for _ in range(TTA):
for i, x_test in enumerate(test_loader):
if(batch % 60 == 0):
print("=", end="")
 x_test[0] = x_test[0].to(device)
 x_test[1] = x_test[1].to(device)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 preds[i*test_loader.batch_size : i*test_loader.batch_size + x_test[0].shape[0]] += z_test
batch += 1
preds /= TTA
del train_ds, val_ds, train_loader, val_loader, x_val, y_val
gc.collect()
preds /= skf.n_splits
return preds, oof
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
 train_text=train_text.apply(remove_emoji)
 test_text=test_text.apply(remove_emoji)
 X_train_embeddings = embed(train_text)
 X_test_embeddings = embed(test_text)
 X_feat=sparse.csr_matrix(X_train_embeddings.numpy())
 X_test=sparse.csr_matrix(X_test_embeddings.numpy())
 X_feat.shape
submission_hub=train_model(X_feat,train_target,X_test,n_estimator=100)
<save_to_csv>
preds, oof = engine()
submission=(0.35*submission_tfidf+0.65*submission_hub ).round().astype(np.int)
 final_submit = pd.DataFrame.from_dict({'id': test['id']})
 final_submit['target']=submission
print(final_submit)
 final_submit.to_csv('submission.csv', index=False)
 print(final_submit['target'].value_counts() )<set_options>
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )
warnings.filterwarnings('ignore')
 %matplotlib inline
sns.set_style('white' )<load_from_csv>
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )
<load_from_csv><EOS> 
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission-v16.csv', index=False )