Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
warnings.filterwarnings("ignore" )
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label<create_dataframe>
df1 = pd.read_csv(".. /input/titanic/train.csv")
 tf1 = pd.read_csv(".. /input/titanic/test.csv")
 result = pd.read_csv(".. /input/titanic/gender_submission.csv" )
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
<define_search_model>
df.isnull().sum()
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<choose_model_class>
tf.isnull().sum()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device )<train_model>
df['Survived'].value_counts()
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc<train_model>
final = pd.concat([df,tf],axis = 0)
 final.drop(['Survived'],axis = 1,inplace = True )
epochs = 35
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))<save_model>
index_NaN_age = list(final["Age"][final["Age"].isnull() ].index)
 for i in index_NaN_age :
age_med = final["Age"].median()
age_pred = final["Age"][(( final['SibSp'] == final.iloc[i]["SibSp"])&(final['Parch'] == final.iloc[i]["Parch"])&(final['Pclass'] == final.iloc[i]["Pclass"])) ].median()
if not np.isnan(age_pred):
final['Age'].iloc[i] = age_pred
else :
final['Age'].iloc[i] = age_med
torch.save(model.state_dict() , "SAKT-HDKIM.pt" )<set_options>
final['Age'].isnull().sum()
del dataset
gc.collect()<split>
final['Age'].fillna(final['Age'].median() ,inplace = True)
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
final['Fare'].isnull().sum()
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
final["Fare"] = final["Fare"].fillna(final["Fare"].median())
import gc
import joblib
import pandas as pd
import numpy as np
import lightgbm as lgb<feature_engineering>
final["Fare"] = final["Fare"].map(lambda n: np.log(n)if n > 0 else 0)
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1<load_pretrained>
new = final['Name'].str.split('.', n=1, expand = True)
 final['First'] = new[0]
final['Last'] = new[1]
new1 = final['First'].str.split(',', n=1, expand = True)
 final['Last Name'] = new1[0]
final['Title'] = new1[1]
new2 = final['Title'].str.split('', n=1, expand = True )
answered_correctly_sum_u_dict = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/answered_correctly_sum_u_dict.pkl.zip")
 count_u_dict = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/count_u_dict.pkl.zip")
 questions_df = pd.read_feather('.. /input/lgbm-with-loop-feature-engineering-dataset/questions_df.feather')
 content_df = pd.read_feather('.. /input/lgbm-with-loop-feature-engineering-dataset/content_df.feather')
 prior_question_elapsed_time_mean = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/prior_question_elapsed_time_mean.pkl.zip" )<define_variables>
final['Title'].value_counts()
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u',
'answered_correctly_avg_c', 'part', 'prior_question_had_explanation',
'prior_question_elapsed_time'
]<load_pretrained>
final.drop(['First','Last','Name','Last Name'],axis = 1,inplace = True )
model = lgb.Booster(model_file=".. /input/lgbm-with-loop-feature-engineering-dataset/fold0_lgb_model.txt")
 model.best_iteration = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/fold0_lgb_model_best_iteration.pkl.zip" )<load_pretrained>
final.replace(to_replace = [ ' Don', ' Rev', ' Dr', ' Mme',
' Major', ' Sir', ' Col', ' Capt',' Jonkheer'], value = ' Honorary(M)', inplace = True)
 final.replace(to_replace = [ ' Ms', ' Lady', ' Mlle',' the Countess', ' Dona'], value = ' Honorary(F)', inplace = True )
optimized_weights = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/optimized_weights.pkl.zip" )<feature_engineering>
df3 = final.copy()
df3 = df3[:891]
df3 = pd.concat([df3,df1['Survived']],axis = 1)
 df3.head()
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()<predict_on_test>
final['Title'].value_counts()
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<merge>
final = pd.get_dummies(final, columns = ["Title"] )
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 preds = optimized_weights[0] * model.predict(test_df[FEATS], num_iteration=400)
 preds += optimized_weights[1] * model.predict(test_df[FEATS], num_iteration=700)
 preds += optimized_weights[2] * model.predict(test_df[FEATS], num_iteration=model.best_iteration)
 test_df[TARGET] = preds
set_predict(test_df[['row_id', TARGET]] )<import_modules>
final["Family"] = final["SibSp"] + final["Parch"] + 1
import glob
import pandas as pd<define_variables>
final['Single'] = final['Family'].map(lambda s: 1 if s == 1 else 0)
 final['SmallF'] = final['Family'].map(lambda s: 1 if s == 2 else 0)
 final['MedF'] = final['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 final['LargeF'] = final['Family'].map(lambda s: 1 if s >= 5 else 0 )
FILES = glob.glob('.. /input/*/prediction_*.csv', recursive=True)
 FILES = [
'.. /input/mysample/tabular_6928.csv',
'.. /input/sub-blend/submission_945_15_folds.csv',
'.. /input/sub-blend/submission_945_5_folds.csv',
'.. /input/melanoma-sub-single-9516/submission_comb(1 ).csv',
'.. /input/train-cv-melanoma/submission.csv',
'.. /input/enet-cv-9430/submission.csv',
'.. /input/mel-9488/submission.csv',
'.. /input/mel-b6-9453/submission.csv'
]<load_from_csv>
final['Embarked'].fillna("S",inplace = True )
sub = pd.read_csv(".. /input/siim-isic-melanoma-classification/sample_submission.csv")
 del sub['target']
w = [0.05, 0.1, 0.1, 0.1, 0.15, 0.15, 0.15, 0.2] 
<define_variables>
final = pd.get_dummies(final, columns = ["Embarked"], prefix="Embarked_from_" )
for counter, f in enumerate(FILES):
print(counter)
 print(f )<load_from_csv>
final.Cabin.isnull().sum()
df = pd.read_csv(f )<feature_engineering>
final.Cabin.value_counts()
df.columns = ['image_name', str(counter)]
df[str(counter)] *= w[counter]
df.head()<merge>
final['Cabin_final'] = df['Cabin'].str[0]
for counter, f in enumerate(FILES):
df = pd.read_csv(f)
 df.columns = ['image_name', str(counter)]
df[str(counter)] *= w[counter]
sub = sub.merge(df, on="image_name" )<prepare_x_and_y>
final['Cabin_final'].fillna('Unknown',inplace = True )
image_name = sub.image_name
sub = sub.drop(columns = ["image_name"])
 target = sub.sum(axis = 1 )<save_to_csv>
final['Cabin_final'].value_counts()
pd.DataFrame({
'image_name' : image_name,
'target' : target
} ).to_csv('submission_b.csv', index=False )<set_options>
final.drop(['Cabin'],axis = 1,inplace = True )
warnings.filterwarnings('ignore' )<install_modules>
final = pd.get_dummies(final, columns = ["Cabin_final"],prefix="Cabin_" )
!pip install -q efficientnet<import_modules>
final.Ticket.value_counts()
import tensorflow as tf
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets<load_from_csv>
final['Ticket'] = final['Ticket'].astype(str)
 final['Ticket_length'] = final.Ticket.apply(len)
 final['Ticket_length'].astype(int)
 final['Ticket_length'].unique()
train = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/train.csv")
 test = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/test.csv")
 sample = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv" )<define_variables>
final['Ticket_length'] = np.where(((final.Ticket_length == 3)|(final.Ticket_length == 4)|(final.Ticket_length == 5)) ,4,final.Ticket_length)
 final['Ticket_length'] = np.where(((final.Ticket_length == 6)) ,5,final.Ticket_length)
 final['Ticket_length'] = np.where(((final.Ticket_length == 7)|(final.Ticket_length == 8)|(final.Ticket_length == 9)|(final.Ticket_length == 10)|(final.Ticket_length == 13)
 |(final.Ticket_length == 17)|(final.Ticket_length == 16)|(final.Ticket_length == 13)|(final.Ticket_length == 12)|(final.Ticket_length == 15)
 |(final.Ticket_length == 11)|(final.Ticket_length == 18)) ,12,final.Ticket_length)
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 GCS_PATH2 = KaggleDatasets().get_gcs_path('malignant-v2-384x384')
 GCS_PATH3 = KaggleDatasets().get_gcs_path('isic2019-384x384')
 filenames_train1 = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 filenames_train2 = tf.io.gfile.glob(GCS_PATH2 + '/train%.2i*.tfrec'%(2*x)for x in range(15))
filenames_train3 = tf.io.gfile.glob(GCS_PATH3 + '/train%.2i*.tfrec'%(2*x)for x in range(15))
filenames_test = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))<randomize_order>
final['Ticket_length'].value_counts()
filenames_train = np.array(filenames_train1+filenames_train2+filenames_train3)
 np.random.shuffle(filenames_train )<feature_engineering>
final['Ticket_length'] = final['Ticket_length'].astype(str)
 final['Ticket_length'] = np.where(((final.Ticket_length == '4')) ,'Below 6',final.Ticket_length)
 final['Ticket_length'] = np.where(((final.Ticket_length == '5')) ,'At 6',final.Ticket_length)
 final['Ticket_length'] = np.where(((final.Ticket_length == '12')) ,'Above 6',final.Ticket_length)
AUTO = tf.data.experimental.AUTOTUNE<set_options>
conversion = pd.get_dummies(final.Ticket_length, prefix = 'Ticket Length')
 final = pd.concat([final , conversion], axis = 1)
 final.drop(['Ticket','Ticket_length'],axis = 1, inplace = True )
cfg = dict(
 batch_size=32,
img_size=384,
lr_start=0.000005,
lr_max=0.00000125,
lr_min=0.000001,
lr_rampup=5,
lr_sustain=0,
lr_decay=0.8,
epochs=12,
transform_prob=1.0,
rot=180.0,
shr=2.0,
hzoom=8.0,
wzoom=8.0,
hshift=8.0,
wshift=8.0,
optimizer='adam',
label_smooth_fac=0.05,
tta_steps=20
)<normalization>
final = pd.get_dummies(final, columns = ["Sex"],prefix="Gender_" )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1], dtype='float32')
 zero = tf.constant([0], dtype='float32')
 rotation_matrix = tf.reshape(
 tf.concat([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0),
[3, 3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0),
[3, 3])
 zoom_matrix = tf.reshape(
 tf.concat([
one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero,
zero, one
],axis=0), [3, 3])
 shift_matrix = tf.reshape(
 tf.concat(
 [one, zero, height_shift, zero, one, width_shift, zero, zero, one],
axis=0), [3, 3])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix)) 
<categorify>
final.head()
final.drop(['PassengerId'],axis = 1,inplace = True)
 final.drop(['SibSp','Parch','Family'],axis = 1,inplace = True )
def transform(image, cfg):
DIM = cfg['img_size']
XDIM = DIM % 2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift)
 x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)
 y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])
 z = tf.ones([DIM * DIM], dtype='int32')
 idx = tf.stack([x, y, z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)
 idx3 = tf.stack([DIM // 2 - idx2[0, ], DIM // 2 - 1 + idx2[1, ]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d, [DIM, DIM, 3] )<prepare_x_and_y>
final.isnull().sum()
def dropout(image, DIM=384, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image<normalization>
correlation = final.copy()
sur = pd.concat([df['Survived'],result['Survived']],axis = 0)
 correlation = pd.concat([correlation,sur],axis = 1 )
def prepare_image(img, cfg=None,droprate=0.5,dropct=8,dropsize=0.2):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['img_size'], cfg['img_size']],
antialias=True)
 img = tf.cast(img, tf.float32)/ 255.0
if cfg['transform_prob'] > tf.random.uniform([1], minval=0, maxval=1):
img = transform(img, cfg)
 if(tf.random.uniform([1], minval=0, maxval=1)> 0.5)&(droprate!=0)&(dropct!=0)&(dropsize!=0):
img = dropout(img, DIM=384, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 return img<prepare_x_and_y>
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn import svm
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from xgboost import XGBClassifier
from sklearn.naive_bayes import BernoulliNB
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score,confusion_matrix
from sklearn import metrics
from sklearn.ensemble import VotingClassifier
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 return example['image'], example['target'] 
<prepare_x_and_y>
x_train = final[:891]
feature_scaler = MinMaxScaler()
x_train = feature_scaler.fit_transform(x_train )
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 return example['image'], example['image_name'] 
<count_values>
y_train = final[891:]
feature_scaler = MinMaxScaler()
y_train = feature_scaler.fit_transform(y_train )
def count_data_items(filenames):
n = [
int(re.compile(r'-([0-9]*)\.' ).search(filename ).group(1))
for filename in filenames
]
return np.sum(n )<create_dataframe>
x_test = df1['Survived']
def getTrainDataset(files, cfg):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.repeat()
ds = ds.shuffle(2048)
 ds = ds.map(lambda img, label: 
(prepare_image(img, cfg=cfg), label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)
 ds = ds.prefetch(AUTO)
 return ds 
<create_dataframe>
y_test = result['Survived']
def getTestDataset(files, cfg, augment=False, repeat=False):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, idnum: 
(prepare_image(img, cfg=cfg), idnum),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
accuracy = []
estimator = []
def getLearnRateCallback(cfg):
lr_start = cfg['lr_start']
lr_max = cfg['lr_max'] * strategy.num_replicas_in_sync * cfg['batch_size']
lr_min = cfg['lr_min']
lr_rampup = cfg['lr_rampup']
lr_sustain = cfg['lr_sustain']
lr_decay = cfg['lr_decay']
def lrfn(epoch):
if epoch < lr_rampup:
lr =(lr_max - lr_start)/ lr_rampup * epoch + lr_start
elif epoch < lr_rampup + lr_sustain:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_rampup -
lr_sustain)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
LR = LogisticRegression()
estimator.append(( 'LR',LogisticRegression()))
cv = cross_val_score(LR,x_train,x_test,cv=10)
 accuracy1 = cv.mean()
accuracy.append(accuracy1)
 print(cv)
 print(cv.mean() )
with strategy.scope() :
model_input = tf.keras.Input(shape=(cfg['img_size'], cfg['img_size'], 3),
name='img_input')
 dummy = tf.keras.layers.Lambda(lambda x: x )(model_input)
 outputs = []
x = efn.EfficientNetB3(include_top=False,
weights='noisy-student',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 x = efn.EfficientNetB4(include_top=False,
weights='noisy-student',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 x = efn.EfficientNetB5(include_top=False,
weights='noisy-student',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.compile(optimizer=cfg['optimizer'],
loss=[
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac']),
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac']),
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac'])
 ],
metrics=[tf.keras.metrics.AUC(name='auc')])
<create_dataframe>
LR.fit(x_train,x_test)
 model1pred = LR.predict(y_train)
 submission1 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission1['PassengerId'] = result['PassengerId']
submission1['Survived'] = model1pred
submission1.to_csv('LogisticRegression(No HT ).csv',index = False )
ds_train = getTrainDataset(filenames_train, cfg ).map(lambda img, label:(img,(label, label, label)))
 stepsTrain = count_data_items(filenames_train)/(cfg['batch_size'] * strategy.num_replicas_in_sync )<train_model>
LR.score(y_train,y_test )
callbacks = [getLearnRateCallback(cfg)]
history = model.fit(ds_train,
validation_data=None,
verbose=1,
steps_per_epoch=stepsTrain,
validation_steps=0,
epochs=10,
callbacks=callbacks )<predict_on_test>
SVC = LinearSVC()
cv = cross_val_score(SVC,x_train,x_test,cv=10)
 accuracy2 = cv.mean()
accuracy.append(accuracy2)
 print(cv)
 print(cv.mean())
steps = count_data_items(filenames_test)/(cfg['batch_size'] * strategy.num_replicas_in_sync)
 z = np.zeros(( cfg['batch_size'] * strategy.num_replicas_in_sync))
ds_testAug = getTestDataset(filenames_test, cfg, augment=True,
repeat=True ).map(lambda img, label:(img,(z, z, z)))
 probs = model.predict(ds_testAug, verbose=1, steps=steps * cfg['tta_steps'] )<concatenate>
SVC.fit(x_train,x_test)
 SVC.score(y_train,y_test )
probs = np.stack(probs)
 probs = probs[:, :count_data_items(filenames_test)* cfg['tta_steps']]
probs = np.stack(np.split(probs, cfg['tta_steps'], axis=1), axis=1)
 probs = np.mean(probs, axis=1 )<save_to_csv>
SVC.fit(x_train,x_test)
 model2pred = SVC.predict(y_train)
 submission2 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission2['PassengerId'] = result['PassengerId']
submission2['Survived'] = model2pred
submission2.to_csv('LinearSVC(No HT ).csv',index = False )
y_test_sorted = np.zeros(( 3, probs.shape[1]))
test = test.reset_index()
test = test.set_index('image_name')
 i = 0
ds_test = getTestDataset(filenames_test, cfg)
 for img, imgid in tqdm(iter(ds_test.unbatch())) :
imgid = imgid.numpy().decode('utf-8')
 y_test_sorted[:, test.loc[imgid]['index']] = probs[:, i, 0]
i += 1
for i in range(y_test_sorted.shape[0]):
submission = sample
submission['target'] = y_test_sorted[i]
submission.to_csv('model_%s.csv' % i, index=False)
 submission = sample
submission['target'] = np.mean(y_test_sorted, axis=0)
 submission.to_csv('ensembled.csv', index=False )<load_pretrained>
poly = svm.SVC(kernel = 'poly', gamma = 'scale')
 cv = cross_val_score(poly,x_train,x_test,cv=10)
 accuracy3 = cv.mean()
accuracy.append(accuracy3)
 print(cv)
 print(cv.mean() )
!gzip model_0.csv
!gzip model_1.csv
!gzip model_2.csv
!gzip ensembled.csv<import_modules>
poly.fit(x_train,x_test)
 poly.score(y_train,y_test )
import numpy as np
import pandas as pd<load_from_csv>
model3pred = poly.predict(y_train)
 submission3 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission3['PassengerId'] = result['PassengerId']
submission3['Survived'] = model3pred
submission3.to_csv('PolynomialSVC(No HT ).csv',index = False )
data1 = pd.read_csv('.. /input/minmax-ensemble-0-9526-lb/submission.csv')
 data2 = pd.read_csv('.. /input/stacking-ensemble-on-my-submissions/submission_mean.csv')
 data3 = pd.read_csv('.. /input/stacking-ensemble-on-my-submissions/submission_median.csv')
 data4 = pd.read_csv('.. /input/analysis-of-melanoma-metadata-and-effnet-ensemble/ensembled.csv')
 data5 = pd.read_csv('.. /input/new-basline-np-log2-ensemble-top-10/submission.csv')
 submission = data1.copy()<feature_engineering>
DT = DecisionTreeClassifier(random_state = 5)
 estimator.append(( 'DT',DecisionTreeClassifier(random_state = 5)))
 cv = cross_val_score(DT,x_train,x_test,cv=10)
 accuracy4 = cv.mean()
accuracy.append(accuracy4)
 print(cv)
 print(cv.mean() )
submission['target'] = 2/6 * data1['target'] + 1/6 * data2['target'] + 1/6 * data3['target'] + 1/6 * data4['target'] + 1/6 * data5['target']<save_to_csv>
DT.fit(x_train,x_test)
 DT.score(y_train,y_test )
submission.to_csv('submission.csv', index=False, float_format='%.6f' )<install_modules>
model4pred = DT.predict(y_train)
 submission4 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission4['PassengerId'] = result['PassengerId']
submission4['Survived'] = model4pred
submission4.to_csv('Decision Tree(No HT ).csv',index = False )
!pip install -q efficientnet
!pip install -q git+https://github.com/AmedeoBiolatti/dsqol<import_modules>
GNB = GaussianNB()
estimator.append(( 'GNB',GaussianNB()))
cv = cross_val_score(GNB,x_train,x_test,cv=10)
 accuracy5 = cv.mean()
accuracy.append(accuracy5)
 print(cv)
 print(cv.mean() )
import os, re, time, tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import metrics, model_selection
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow import keras
from tensorflow.keras import backend as K
from efficientnet import tfkeras as efnet
from kaggle_datasets import KaggleDatasets<import_modules>
GNB.fit(x_train,x_test)
 GNB.score(y_train,y_test )
from dsqol.tf import imgaug
from dsqol.tf.data import balance
from dsqol.tf.utils import average
from dsqol.tf import losses<init_hyperparams>
model5pred = GNB.predict(y_train)
 submission5 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission5['PassengerId'] = result['PassengerId']
submission5['Survived'] = model5pred
submission5.to_csv('Gaussian NB(No HT ).csv',index = False )
SEED = 42
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 TIME_BUDGET = 2.5 * 3600
FOLDS = 5
INCLUDE_2019 = 0
INCLUDE_2018 = 1
INCLUDE_MALIGNANT = 0
IMG_READ_SIZE = 512
IMG_SIZE = 512
BALANCE_POS_RATIO = 0.08
EFF_NET = 5
LOSS_TYPE = 'BCE'
LOSS_PARAMS = dict(label_smoothing=0.05)
 BATCH_SIZE = 32
EPOCHS = 20
TBM = 6
TTA = 15
VALID_FREQ = 1
N_SWA = 3
SWA_DECAY = 0.9<train_on_grid>
MNB = MultinomialNB()
estimator.append(( 'MNB',MultinomialNB()))
cv = cross_val_score(MNB,x_train,x_test,cv=10)
 accuracy6 = cv.mean()
accuracy.append(accuracy6)
 print(cv)
 print(cv.mean() )
DEVICE = "TPU"
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
GLOBAL_BATCH_SIZE = BATCH_SIZE * REPLICAS
print("REPLICAS: %d" % REPLICAS )<define_variables>
MNB.fit(x_train,x_test)
 MNB.score(y_train,y_test )
GCS_PATH1 = KaggleDatasets().get_gcs_path('melanoma-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
GCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
GCS_PATH3 = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))<load_from_csv>
MNB.fit(x_train,x_test)
 model6pred = MNB.predict(y_train)
 submission6 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission6['PassengerId'] = result['PassengerId']
submission6['Survived'] = model6pred
submission6.to_csv('MultinomialNB(No HT ).csv',index = False )
df_base_train = pd.read_csv(".. /input/siim-isic-melanoma-classification/train.csv")
 df_base_test = pd.read_csv(".. /input/siim-isic-melanoma-classification/test.csv" )<define_variables>
RF = RandomForestClassifier(random_state = 5)
 estimator.append(( 'RF',RandomForestClassifier(random_state = 5)))
 cv = cross_val_score(RF,x_train,x_test,cv=10)
 accuracy7 = cv.mean()
accuracy.append(accuracy7)
 print(cv)
 print(cv.mean() )
train_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, "train*.tfrec"))
if INCLUDE_2019:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, "train%.2i*.tfrec" % i)for i in range(1, 30, 2)])
 if INCLUDE_2018:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, "train%.2i*.tfrec" % i)for i in range(0, 30, 2)])
 if INCLUDE_MALIGNANT:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH3, "train%.2i*.tfrec" % i)for i in range(15, 30, 1)])
 print("%d train files found" % len(train_files))<train_model>
RF.fit(x_train,x_test)
 RF.score(y_train,y_test )
test_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, "test*.tfrec"))
print("%d test files found" % len(test_files))<prepare_x_and_y>
RF.fit(x_train,x_test)
 model7pred = RF.predict(y_train)
 submission7 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission7['PassengerId'] = result['PassengerId']
submission7['Survived'] = model7pred
submission7.to_csv('RandomForest(No HT ).csv',index = False )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name=True):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
GBC = GradientBoostingClassifier(random_state = 5)
 estimator.append(( 'GBC',GradientBoostingClassifier(random_state = 5)))
 cv = cross_val_score(GBC,x_train,x_test,cv=10)
 accuracy8 = cv.mean()
accuracy.append(accuracy8)
 print(cv)
 print(cv.mean() )
def dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0):
return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
def ClasswiseMixup(probability: float=1.0, batch_size: int = 16, alpha: float=1.0, beta: float=1.0):
def classwise_mixup(img, tar):
do = tf.cast(tf.random.uniform([batch_size], 0, 1)< probability, tf.float32)
 gamma = tf.reshape(tf.random.uniform([batch_size], 0, 1)* do, [-1, 1, 1, 1])
 shuffled_img = tf.random.shuffle(img)
 img_new =(1 - gamma)* img + gamma * shuffled_img
return img_new
return classwise_mixup<statistical_test>
GBC.fit(x_train,x_test)
 GBC.score(y_train,y_test )
AUG_BS = 64
def base_aug(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 return img
dropout_aug = lambda img, o: dropout(img, DIM=IMG_READ_SIZE, PROBABILITY=0.75, CT=8, SZ=0.15)
 transform_aug = imgaug.Transform(dim=IMG_READ_SIZE, hzoom_mult=10.0, wzoom_mult=10.0)
 cw_mixup_aug = ClasswiseMixup(probability=0.1, batch_size=AUG_BS)
 def basic_augmentation_pipeline(ds: tf.data.Dataset, dim=None, batch_size=None)-> tf.data.Dataset:
ds = ds.map(lambda i, o:(transform_aug(i), o), num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(base_aug(i), o), num_parallel_calls=AUTO)
 return ds<create_dataframe>
GBC.fit(x_train,x_test)
 model8pred = GBC.predict(y_train)
 submission8 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission8['PassengerId'] = result['PassengerId']
submission8['Survived'] = model8pred
submission8.to_csv('GradientBoosting(No HT ).csv',index = False )
def get_dataset(files, augment=False, repeat=False, shuffle=False, labeled=True, batch_size=16, drop_remainder=False,
dim=256, read_dim=None 
)-> tf.data.Dataset:
if read_dim is None:
read_dim = dim
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024 * 8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 if augment:
ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim)
 if isinstance(augment, list):
for a in augment:
ds = ds.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)
 ds = ds.batch(batch_size, drop_remainder=drop_remainder)
 ds = ds.prefetch(AUTO)
 return ds
def get_balanced_dataset(files, augment=False, cw_augment=False, repeat=False, shuffle=False, batch_size=16, drop_remainder=False,
dim=256, read_dim=None, pos_ratio=False)-> tf.data.Dataset:
if read_dim is None:
read_dim = dim
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds0, ds1 = balance.separate_by_target(ds)
 ds0 = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 ds1 = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 if cw_augment:
ds0 = ds0.batch(AUG_BS, drop_remainder=drop_remainder); ds1 = ds1.batch(AUG_BS, drop_remainder=drop_remainder)
 for a in cw_augment:
ds0 = ds0.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds1 = ds1.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds0 = ds0.unbatch() ; ds1 = ds1.unbatch()
ds = balance.merge_ds(ds0, ds1, pos_ratio)
 if shuffle:
ds = ds.shuffle(1024)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if augment:
ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim)
 if isinstance(augment, list):
for a in augment:
ds = ds.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)
 ds = ds.batch(batch_size, drop_remainder=drop_remainder)
 ds = ds.prefetch(AUTO)
 return ds<prepare_x_and_y>
XGB = XGBClassifier(random_state = 5)
 estimator.append(( 'XGB', XGBClassifier(random_state = 5)))
 cv = cross_val_score(XGB,x_train,x_test,cv=10)
 accuracy9 = cv.mean()
accuracy.append(accuracy9)
 print(cv)
 print(cv.mean() )
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 pass<create_dataframe>
XGB.fit(x_train,x_test)
 XGB.score(y_train,y_test )
show_dataset(128, 8, 2, get_balanced_dataset(train_files, augment=[dropout_aug], cw_augment=[cw_mixup_aug] ).take(10 ).unbatch() )<choose_model_class>
XGB.fit(x_train,x_test)
 model9pred = XGB.predict(y_train)
 submission9 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission9['PassengerId'] = result['PassengerId']
submission9['Survived'] = model9pred
submission9.to_csv('XGBoosting(No HT ).csv',index = False )
def build_model(dim=128, ef=0):
inp = keras.layers.Input(shape=(dim,dim,3))
base = getattr(efnet, 'EfficientNetB%d' % ef )(input_shape=(dim, dim, 3), weights='imagenet', include_top=False)
 x = base(inp)
 x = keras.layers.GlobalAveragePooling2D()(x)
 x = keras.layers.Dense(1 )(x)
 x = keras.layers.Activation('sigmoid', dtype='float32' )(x)
 model = keras.Model(inputs=inp,outputs=x)
 opt = keras.optimizers.Adam(learning_rate=1e-3)
 if LOSS_TYPE.upper() == 'BCE':
loss = keras.losses.BinaryCrossentropy(**LOSS_PARAMS)
 elif LOSS_TYPE.upper() == 'FOCAL':
loss = losses.BinaryFocalLoss(**LOSS_PARAMS)
 model.compile(optimizer=opt, loss=loss, metrics=['AUC'])
 return model<init_hyperparams>
KNN = KNeighborsClassifier(n_neighbors = 11)
 estimator.append(( 'KNN',KNeighborsClassifier(n_neighbors = 11)))
 cv = cross_val_score(KNN,x_train,x_test,cv=10)
 accuracy10 = cv.mean()
accuracy.append(accuracy10)
 print(cv)
 print(cv.mean() )
mult = 1
lr_start = 5e-6
lr_max = 1.25e-6 * GLOBAL_BATCH_SIZE
lr_min = 1e-6
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr * mult
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
 plt.plot([lrfn(e)for e in range(EPOCHS)])
 plt.xlabel('Epoch'); plt.ylabel('LR')
 plt.subplot(1, 2, 2)
 plt.plot([lrfn(e)for e in range(EPOCHS)])
 plt.xlabel('Epoch'); plt.ylabel('Log LR')
 plt.yscale('log');<load_from_csv>
KNN.fit(x_train,x_test)
 KNN.score(y_train,y_test )
CKPT_FOLDER = ".. /working/ckpt"
if not os.path.exists(CKPT_FOLDER):
os.mkdir(CKPT_FOLDER)
 folds = list(model_selection.KFold(n_splits=FOLDS, shuffle=True, random_state=SEED ).split(np.arange(15)))
 testiness = pd.read_csv(".. /input/spicv-spicy-vi-make-your-cv-more-testy/testiness.csv")
 TOTAL_POS = 581 + 2858 * INCLUDE_2019 + 1651 * INCLUDE_2018 + 580 * INCLUDE_MALIGNANT<create_dataframe>
KNN.fit(x_train,x_test)
 model10pred = KNN.predict(y_train)
 submission10 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission10['PassengerId'] = result['PassengerId']
submission10['Survived'] = model10pred
submission10.to_csv('KNN(No HT ).csv',index = False )
VERBOSE = 1
PLOT = 1
histories = []
df_oof = pd.DataFrame() ; df_res = pd.DataFrame()
t_start = time.time()
for fold,(idTrain, idValid)in enumerate(folds):
print("
print(( "
print("
if DEVICE == 'TPU':
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 fold_valid_files = [f for f in train_files if any([int(re.match("^train([0-9]+)", f.split("/")[-1] ).group(1)) % 15 == i for i in idValid])]
fold_valid_files = [f for f in fold_valid_files if GCS_PATH1 in f]
fold_train_files = [f for f in train_files if f not in fold_valid_files]
np.random.shuffle(fold_train_files)
 print("Train files: %d\t\t Valid files: %d" %(len(fold_train_files), len(fold_valid_files)))
 K.clear_session()
if DEVICE == 'TPU':
keras.mixed_precision.experimental.set_policy('mixed_bfloat16')
 with strategy.scope() :
model = build_model(dim=IMG_SIZE, ef=EFF_NET)
 FOLD_CKPT_FOLDER = os.path.join(CKPT_FOLDER, "fold%d" % fold)
 if not os.path.exists(FOLD_CKPT_FOLDER):
os.mkdir(FOLD_CKPT_FOLDER)
 callbacks =[
keras.callbacks.ModelCheckpoint(os.path.join(FOLD_CKPT_FOLDER, "model_fold%d_e{epoch:02d}.h5" % fold), save_weights_only=True),
keras.callbacks.LearningRateScheduler(lrfn)
 ]
if BALANCE_POS_RATIO:
print("Using balanced dataset with pos_ratio = %d%%" % int(100 * BALANCE_POS_RATIO))
ds_train = get_balanced_dataset(fold_train_files, repeat=True, augment=[dropout_aug], drop_remainder=True, shuffle=True,
pos_ratio=BALANCE_POS_RATIO,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)
 FOLD_POS = TOTAL_POS *(FOLDS - 1)/ FOLDS
STEPS = int(FOLD_POS / BALANCE_POS_RATIO / GLOBAL_BATCH_SIZE)
 else:
print("Using unbalanced dataset")
 ds_train = get_dataset(fold_train_files, repeat=True, augment=[dropout_aug], drop_remainder=True, shuffle=True,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)
 STEPS = int(count_data_items(fold_train_files)/ GLOBAL_BATCH_SIZE)
 ds_valid = get_dataset(fold_valid_files, repeat=False, augment=False, drop_remainder=False, shuffle=False,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM)
 print("Training...")
 history = model.fit(
 ds_train,
validation_data = ds_valid,
epochs = EPOCHS,
steps_per_epoch = STEPS,
verbose = VERBOSE,
callbacks = callbacks,
validation_freq = VALID_FREQ 
)
 histories.append(history)
 ckpt_files = np.sort(tf.io.gfile.glob(os.path.join(FOLD_CKPT_FOLDER, "*.h5")))
 ckpt_files_fow_swa = ckpt_files[-N_SWA:]
if len(ckpt_files_fow_swa)> 1:
with strategy.scope() :
model = average.average_weights(ckpt_files_fow_swa, decay=SWA_DECAY, model=model)
 for f in ckpt_files:
os.remove(f)
 model.save(os.path.join(CKPT_FOLDER, "model_fold%d.h5" % fold))
ds_valid = get_dataset(fold_valid_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True)
 ct_valid = count_data_items(fold_valid_files); STEPS = int(np.ceil(TTA * ct_valid / GLOBAL_BATCH_SIZE / TBM))
fold_valid_pred = model.predict(ds_valid, steps=STEPS, verbose=1)
 fold_valid_pred = fold_valid_pred[:ct_valid * TTA,]
ds_valid = get_dataset(fold_valid_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)
 fold_valid_names = np.concatenate([np.array([ni.decode("utf-8")for ni in n.numpy() ])for n in ds_valid.map(lambda i, n: n)], 0)
 fold_df = pd.DataFrame({'image_name': np.tile(fold_valid_names, [TTA]), 'pred': fold_valid_pred.squeeze() , 'fold': fold})
 df_oof = pd.concat([df_oof, fold_df])
 fold_df['image_name'] = fold_df['image_name'].str.replace('_downsampled', '')
 fold_df = fold_df.groupby('image_name' ).mean().reset_index()
fold_df = fold_df.merge(df_base_train[['image_name', 'patient_id', 'target']], on='image_name' ).merge(testiness, on='image_name')
 fold_df['fold'] = fold
auc = metrics.roc_auc_score(fold_df.target, fold_df.pred)
 ds_test = get_dataset(test_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True, labeled=False)
 ct_test = count_data_items(test_files); STEPS = int(np.ceil(TTA * ct_test / GLOBAL_BATCH_SIZE / TBM))
fold_test_pred = model.predict(ds_test.map(lambda i, l: i), steps=STEPS, verbose=1)
 fold_test_pred = fold_test_pred[:ct_test * TTA,]
ds_test = get_dataset(test_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)
 fold_test_names = np.concatenate([np.array([ni.decode("utf-8")for ni in n.numpy() ])for n in ds_test.map(lambda i, n: n)], 0)
 fold_res = pd.DataFrame({'image_name': np.tile(fold_test_names, [TTA]), 'pred': fold_test_pred.squeeze() , 'fold': fold})
 df_res = pd.concat([df_res, fold_res])
 used_time_till_now = time.time() - t_start
time_per_fold = used_time_till_now /(fold + 1)
 print("Validation AUC last epoch = %.4f" % history.history['val_auc'][-1])
 print("Validation AUC(TTA %2d)= %.4f" %(TTA, auc))
print("Total time = %ds\t\tTime per fold = %ds" %(int(used_time_till_now), int(time_per_fold)))
 if PLOT:
plt.figure(figsize=(16, 4))
plt.subplot(1, 2, 1)
 plt.plot(history.history['loss'], color='tab:blue', marker='o')
 plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_loss'], color='tab:blue', marker='x', linestyle=':')
 plt.yscale('log')
 plt.subplot(1, 2, 2)
 plt.plot(history.history['auc'], color='tab:red', marker='o')
 plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_auc'], color='tab:red', marker='x', linestyle=':')
 plt.show()
del model, ds_train, ds_valid, ds_test
print("
")
 if(fold + 1)< FOLDS:
time_till_next_fold = used_time_till_now + time_per_fold
if time_till_next_fold > TIME_BUDGET:
print("Forecasted total time till next fold completion = %d(budget = %d)" %(time_till_next_fold, TIME_BUDGET))
break
pass<create_dataframe>
Models = ['Logistic Regression','Linear SVM','Polynomial SVM','Decision Tree','Gaussian NB','Multinomial NB','Random Forest Classifier','Gradient Boost Classifier','XG Boosting','K-Nearest Neighbors']
total = list(zip(Models,accuracy))
output1 = pd.DataFrame(total, columns = ['Models','Accuracy'])
<merge>
vot_soft = VotingClassifier(estimators = estimator, voting ='soft')
 vot_soft.fit(x_train, x_test)
 y_pred = vot_soft.predict(y_train)
 vot_soft.score(y_train,y_test )
xxx = df_oof.groupby('image_name' ).mean().reset_index().merge(df_base_train, on='image_name')
 print("OOF AUC(TTA %d)= %.4f" %(TTA, metrics.roc_auc_score(xxx.target, xxx.pred)) )<save_to_csv>
modelpred1 = vot_soft.predict(y_train)
 sub1 = pd.DataFrame(columns = ['PassengerId','Survived'])
 sub1['PassengerId'] = result['PassengerId']
sub1['Survived'] = modelpred1
sub1.to_csv('SoftVoting(NO HT ).csv',index = False )
df_res.to_csv('.. /working/test_res_all.csv', index=False)
 df_oof.to_csv('.. /working/oof_res_all.csv', index=False )<save_to_csv>
vot_hard = VotingClassifier(estimators = estimator, voting ='hard')
 vot_hard.fit(x_train, x_test)
 y_pred = vot_hard.predict(y_train)
 vot_hard.score(y_train,y_test )
df_res[['image_name', 'pred']].groupby('image_name' ).mean().reset_index().rename({'pred': 'target'}, axis=1 ).to_csv("submission_6_20.csv", index=False )<import_modules>
modelpred2 = vot_hard.predict(y_train)
 sub2 = pd.DataFrame(columns = ['PassengerId','Survived'])
 sub2['PassengerId'] = result['PassengerId']
sub2['Survived'] = modelpred2
sub2.to_csv('HardVoting(NO HT ).csv',index = False )
import numpy as np
import pandas as pd
import scipy.stats
import matplotlib.pyplot as plt
from scipy.stats import spearmanr
from matplotlib.colors import LogNorm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf<set_options>
Accuracy = []
Estimator = []
orange_black = [
'
]
plt.style.use('ggplot' )<load_from_csv>
n_tta = 1500
file = 'tta-exploration-128x128-b0'
all_tta_test = pd.read_csv('.. /input/'+file+'/all_tta_test.csv')
 all_tta_test.columns.values[0] = 'image_name'
submission = pd.read_csv('.. /input/'+file+'/submission.csv')
 tta_keys = [str(i)for i in range(n_tta)]
oof = pd.read_csv('.. /input/'+file+'/oof.csv' )<concatenate>
lr = LogisticRegression(C = 100,penalty = 'l2', solver = 'newton-cg',class_weight = 'dict', max_iter = 900)
 Estimator.append(( 'lr',LogisticRegression(C = 1,penalty = 'l2', solver = 'newton-cg',class_weight = 'dict', max_iter = 900)))
 cv = cross_val_score(lr,x_train,x_test,cv=10)
 Accuracy1 = cv.mean()
Accuracy.append(Accuracy1)
 print(cv)
 print(cv.mean() )
def IQR(data,ax=1):
return np.subtract(*np.percentile(data, [75, 25],axis=ax))<define_variables>
lr.fit(x_train,x_test)
 lr.score(y_train,y_test )
tta_arr = np.array(tta_preds)
 n_samp = tta_arr.shape[1]
average_t=[];max_t=[];std_t=[]
error_std=[]
for TTA in range(2,100):
end = int(n_tta//TTA)
 samples = tta_arr[:,:TTA*end].reshape(-1,TTA)
 all_avs = np.mean(samples,axis=1 ).reshape(len(tta_arr),end)
 if all_avs.shape[1]>10:
all_stds = np.std(samples,axis=1 ).reshape(len(tta_arr),end)
 all_e_std = np.std(all_stds,axis=1)
 error_std+=[np.mean(all_e_std/std_pred)]
if all_avs.shape[1] > 30:
std_errors = np.std(all_avs,axis=1)
 else:
std_errors = np.mean(all_stds/np.sqrt(TTA),axis=1)
 std_errors = std_errors/mn_pred
std_t+=[np.std(std_errors)]
average_t+=[np.mean(std_errors)]
max_t+=[np.max(std_errors)] 
<compute_test_metric>
model11pred = lr.predict(y_train)
 submission11 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission11['PassengerId'] = result['PassengerId']
submission11['Survived'] = model11pred
submission11.to_csv('LogisticRegression(HT ).csv',index = False )
tta_preds = oof[tta_keys]
mns = dict(
 amean = np.mean(tta_preds,axis=1),
gmean = scipy.stats.gmean(tta_preds,axis=1),
median = np.median(tta_preds,axis=1)
)
 for mn in mns:
print('{} AUC = {}'.format(mn,roc_auc_score(oof['target'],mns[mn])) )<save_to_csv>
def make_sub(func,name='submission'):
submission = pd.DataFrame(dict(image_name=test_tta['0'], target=func(preds_all,axis=1)))
 submission.to_csv(name+'.csv', index=False)
 submission.head()
test_tta = pd.read_csv('.. /input/tta-exploration-128x128-b0/all_tta_test.csv')
 FOLDS=3; n_test_tta=11
preds_all = [np.mean(test_tta[[str(i)for i in range(j*n_test_tta,(j+1)*n_test_tta)]],axis=1)for j in range(FOLDS)]
preds_all = np.vstack(preds_all ).T
make_sub(np.mean,name='amean_submission')
 make_sub(scipy.stats.gmean,name='gmean_submission')
 make_sub(np.median,name='median_submission' )<load_from_csv>
svc = LinearSVC(C = 0.1,penalty = 'l2', loss = 'hinge',class_weight = 'balanced')
 cv = cross_val_score(svc,x_train,x_test,cv=10)
 Accuracy2 = cv.mean()
Accuracy.append(Accuracy2)
 print(cv)
 print(cv.mean() )
n_tta = 11
FOLDS=3
files = ['tta-exploration-128x128-b0','tta-exploration-128x128-b3']
all_tta_list = [pd.read_csv('.. /input/'+file+'/all_tta_test.csv')for file in files]
for df in all_tta_list:
df.columns.values[0] = 'image_name'
sub_list= [pd.read_csv('.. /input/'+file+'/submission.csv')for file in files]
tta_keys = [str(i)for i in range(n_tta)]
oof_list = [pd.read_csv('.. /input/'+file+'/oof.csv')for file in files]<drop_column>
svc.fit(x_train,x_test)
 svc.score(y_train,y_test )
def set_right(df,tta_keys):
df['std']=np.std(df[tta_keys],axis=1)
 return df.drop(tta_keys,axis=1)
 oof1 = [set_right(df,tta_keys ).set_index('image_name')for df in oof_list]<concatenate>
model12pred = svc.predict(y_train)
 submission12 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission12['PassengerId'] = result['PassengerId']
submission12['Survived'] = model12pred
submission12.to_csv('SVCLinear(HT ).csv',index = False )
df_oof = pd.concat(oof1,axis=1)
 labels = df_oof['target'].iloc[:,0]<find_best_params>
roc=[]
WGTS=np.linspace(0,1,11)
 for wgt in WGTS:
pred = wgt*df_oof['pred'].iloc[:,0] +(1-wgt)*df_oof['pred'].iloc[:,1]
roc+=[roc_auc_score(labels,pred)]
best_roc = max(roc)
 loc = np.where(roc==best_roc)[0][0]
best_weight = WGTS[loc]
print('The best roc = {:.6f} with weights :({},{})'.format(best_roc, best_weight,(1-best_weight)) )<import_modules>
SVM_all = svm.SVC(C = 1,degree = 2, kernel = 'poly',class_weight = 'balanced',gamma = 'scale')
 cv = cross_val_score(svc,x_train,x_test,cv=10)
 Accuracy3 = cv.mean()
Accuracy.append(Accuracy3)
 print(cv)
 print(cv.mean() )
from xgboost import XGBClassifier<prepare_x_and_y>
SVM_all.fit(x_train,x_test)
 SVM_all.score(y_train,y_test )
ids = df_oof.index
fold_finder = df_oof['fold'].iloc[:,0]
def get_oof(DATA,LABELS,folds=10,PRINT=0,SEED=42):
skf = StratifiedKFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
for fold,(idxT,idxV)in enumerate(skf.split(DATA,LABELS)) :
X = DATA[idxT]; y = LABELS[idxT]
X_val = DATA[idxV]; y_val = LABELS[idxV]
model = XGBClassifier(objective='binary:logistic',
seed=0,
nthread=-1,
learning_rate=0.01,
n_estimators=1200,
scale_pos_weight=1,
min_child_weight=1,
max_depth=2,
subsample=0.7,
colsample_bytree=0.9,
gamma=5,
random_state=42, 
)
 model.fit(X,y)
 pred = model.predict_proba(X_val)[:,1]
oof_pred.append(pred)
 oof_tar.append(y_val)
 oof_names.append(ids[idxV])
 oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 if PRINT:
print('
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.5f'%auc )<categorify>
model13pred = SVM_all.predict(y_train)
 submission13 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission13['PassengerId'] = result['PassengerId']
submission13['Survived'] = model13pred
submission13.to_csv('PolynomialSVM(HT ).csv',index = False )
FOLDS = 20
print('Including confidence estimate:')
 get_oof(np.concatenate(( df_oof['pred'],df_oof['std']),axis=1),labels,folds=FOLDS)
 print('
Only predictions:')
 get_oof(np.array(df_oof['pred']),labels,folds=FOLDS )<set_options>
warnings.filterwarnings('ignore')
 seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val )<set_options>
dt = DecisionTreeClassifier(class_weight = 'balanced',criterion = 'entropy',max_depth = 5,min_samples_split = 2,splitter = 'best',random_state = 6)
 Estimator.append(( 'dt',DecisionTreeClassifier(class_weight = 'balanced',criterion = 'entropy',max_depth = 5,min_samples_split = 2,splitter = 'best',random_state = 6)))
 cv = cross_val_score(dt,x_train,x_test,cv=10)
 Accuracy4 = cv.mean()
Accuracy.append(Accuracy4)
 print(cv)
 print(cv.mean() )
black_red = [
'
]
plt.style.use('fivethirtyeight' )<load_from_csv>
dt.fit(x_train,x_test)
 dt.score(y_train,y_test )
train = pd.read_csv('.. /input/melanomaextendedtabular/external_upsampled_tabular.csv')
 test = pd.read_csv('.. /input/melanomaextendedtabular/test_tabular.csv')
 sample = pd.read_csv('.. /input/melanomaextendedtabular/sample_submission.csv' )<rename_columns>
model14pred = SVM_all.predict(y_train)
 submission14 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission14['PassengerId'] = result['PassengerId']
submission14['Survived'] = model14pred
submission14.to_csv('DecisionTrees(HT ).csv',index = False )
train.columns = [
'img_name', 'sex', 'age', 'location', 'target','width','height'
]
test.columns = ['img_name', 'sex', 'age', 'location','width','height'] 
<categorify>
for df in [train, test]:
df['location'].fillna('unknown', inplace=True)
 train['sex'].fillna('unknown', inplace=True)
 train['age'].fillna(-1, inplace=True )<statistical_test>
mnb = MultinomialNB(alpha = 1,fit_prior = True)
 Estimator.append(( 'mnb',MultinomialNB(alpha = 1,fit_prior = True)))
 cv = cross_val_score(mnb,x_train,x_test,cv=10)
 Accuracy5 = cv.mean()
Accuracy.append(Accuracy5)
 print(cv)
 print(cv.mean() )
ids_train = train.location.values
ids_test = test.location.values
ids_train_set = set(ids_train)
 ids_test_set = set(ids_test)
 location_not_overlap = list(ids_train_set.symmetric_difference(ids_test_set))
n_overlap = len(location_not_overlap)
 if n_overlap == 0:
print(
 f'There are no different body parts occuring between train and test set...' 
)
 else:
print('There are some non-overlapping values between train and test set!
')
 print(f'Different ones are:
{pd.Series(np.setdiff1d(( train.location.value_counts().index), pd.Series(test.location.value_counts().index)))}')
<categorify>
mnb.fit(x_train,x_test)
 mnb.score(y_train,y_test )
train.replace(['anterior torso','lateral torso','posterior torso'], 'torso', inplace=True )<data_type_conversions>
model15pred = mnb.predict(y_train)
 submission15 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission15['PassengerId'] = result['PassengerId']
submission15['Survived'] = model15pred
submission15.to_csv('MultinomialNB(HT ).csv',index = False )
train['res']= train['width'].astype(str)+'x'+train['height'].astype(str)
 test['res']= test['width'].astype(str)+'x'+test['height'].astype(str )<drop_column>
train.drop(['res'], axis=1, inplace=True)
 test.drop(['res'], axis=1, inplace=True )<categorify>
sex_dummies = pd.get_dummies(train['sex'], prefix='sex')
 train = pd.concat([train, sex_dummies], axis=1)
 sex_dummies = pd.get_dummies(test['sex'], prefix='sex')
 test = pd.concat([test, sex_dummies], axis=1)
 train.drop(['sex'], axis=1, inplace=True)
 test.drop(['sex'], axis=1, inplace=True )<categorify>
rf = RandomForestClassifier(oob_score = True,n_estimators =650 ,min_samples_split = 4,max_features = 'log2',max_depth =6,criterion = 'gini',class_weight = 'balanced_subsample',bootstrap = True)
 Estimator.append(( 'rf',RandomForestClassifier(oob_score = True,n_estimators =650 ,min_samples_split = 4,max_features = 'log2',max_depth =6,criterion = 'gini',class_weight = 'balanced_subsample',bootstrap = True)))
 cv = cross_val_score(rf,x_train,x_test,cv=10)
 Accuracy6 = cv.mean()
Accuracy.append(Accuracy6)
 print(cv)
 print(cv.mean() )
anatom_dummies = pd.get_dummies(train['location'], prefix='anatom')
 train = pd.concat([train, anatom_dummies], axis=1)
 anatom_dummies = pd.get_dummies(test['location'], prefix='anatom')
 test = pd.concat([test, anatom_dummies], axis=1)
 train.drop('location', axis=1, inplace=True)
 test.drop(['location'], axis=1, inplace=True )<drop_column>
rf.fit(x_train,x_test)
 rf.score(y_train,y_test )
for df in [train, test]:
df.drop('img_name', axis=1, inplace=True )<import_modules>
model16pred = rf.predict(y_train)
 submission16 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission16['PassengerId'] = result['PassengerId']
submission16['Survived'] = model16pred
submission16.to_csv('RandomForest(HT ).csv',index = False )
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold, train_test_split, cross_val_score, cross_validate
from sklearn.metrics import roc_auc_score, roc_curve<prepare_x_and_y>
X = train.drop('target', axis=1)
 y = train.target<split>
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.15,
stratify=y,
random_state=42)
 cv = StratifiedKFold(5, shuffle=True, random_state=42 )<choose_model_class>
gbc = GradientBoostingClassifier(loss = 'exponential',n_estimators =200 ,min_samples_split = 4,max_features = 'auto',max_depth =9,learning_rate =.01,subsample =.1)
 Estimator.append(( 'gbc',GradientBoostingClassifier(loss = 'exponential',n_estimators =200 ,min_samples_split = 4,max_features = 'auto',max_depth =9,learning_rate =.01,subsample =.1)))
 cv = cross_val_score(gbc,x_train,x_test,cv=10)
 Accuracy7 = cv.mean()
Accuracy.append(Accuracy7)
 print(cv)
 print(cv.mean() )
xg = xgb.XGBClassifier(
 n_estimators=750,
min_child_weight=0.81,
learning_rate=0.025,
max_depth=2,
subsample=0.80,
colsample_bytree=0.42,
gamma=0.10,
random_state=42,
n_jobs=-1,
)<define_variables>
gbc.fit(x_train,x_test)
 gbc.score(y_train,y_test )
estimators = [xg]<compute_train_metric>
model17pred = gbc.predict(y_train)
 submission17 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission17['PassengerId'] = result['PassengerId']
submission17['Survived'] = model17pred
submission17.to_csv('GradientBoosting(HT ).csv',index = False )
def model_check(X_train, y_train, estimators, cv):
model_table = pd.DataFrame()
row_index = 0
for est in estimators:
MLA_name = est.__class__.__name__
model_table.loc[row_index, 'Model Name'] = MLA_name
cv_results = cross_validate(est,
X_train,
y_train,
cv=cv,
scoring='roc_auc',
return_train_score=True,
n_jobs=-1)
 model_table.loc[row_index,
'Train roc Mean'] = cv_results['train_score'].mean()
model_table.loc[row_index,
'Test roc Mean'] = cv_results['test_score'].mean()
model_table.loc[row_index, 'Test Std'] = cv_results['test_score'].std()
model_table.loc[row_index, 'Time'] = cv_results['fit_time'].mean()
row_index += 1
model_table.sort_values(by=['Test roc Mean'],
ascending=False,
inplace=True)
 return model_table<compute_train_metric>
xg.fit(X_train, y_train)
 validation = xg.predict_proba(X_test)[:, 1]
roc_auc_score(y_test, validation )<prepare_x_and_y>
adv_train = train.copy()
adv_train.drop('target', axis=1, inplace=True)
 adv_test = test.copy()
adv_train['dataset_label'] = 0
adv_test['dataset_label'] = 1
adv_master = pd.concat([adv_train, adv_test], axis=0)
 adv_X = adv_master.drop('dataset_label', axis=1)
 adv_y = adv_master['dataset_label']<split>
xgb = XGBClassifier(colsample_bytree =.6,eta = 0.5,gamma = 1,max_depth = 5,min_child_weight = 6,subsample = 1)
 Estimator.append(( 'xgb',XGBClassifier(colsample_bytree =.6,eta = 0.5,gamma = 1,max_depth = 5,min_child_weight = 6,subsample = 1)))
 cv = cross_val_score(xgb,x_train,x_test,cv=10)
 Accuracy8 = cv.mean()
Accuracy.append(Accuracy8)
 print(cv)
 print(cv.mean() )
adv_X_train, adv_X_test, adv_y_train, adv_y_test = train_test_split(adv_X,
adv_y,
test_size=0.4,
stratify=adv_y,
random_state=42 )<train_model>
xgb.fit(x_train,x_test)
 gbc.score(y_train,y_test )
xg_adv = xgb.XGBClassifier(
 random_state=42,
n_jobs=-1, 
)
 xg_adv.fit(adv_X_train, adv_y_train)
 validation = xg_adv.predict_proba(adv_X_test)[:,1]<split>
model18pred = xgb.predict(y_train)
 submission18 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission18['PassengerId'] = result['PassengerId']
submission18['Survived'] = model18pred
submission18.to_csv('XGBoosting(HT ).csv',index = False )
adv_X.drop(['sex_unknown', 'height', 'width'], axis=1, inplace=True)
 adv_X_train, adv_X_test, adv_y_train, adv_y_test = train_test_split(adv_X,
adv_y,
test_size=0.4,
stratify=adv_y,
random_state=42)
 xg_adv.fit(adv_X_train, adv_y_train)
 validation = xg_adv.predict_proba(adv_X_test)[:,1]<drop_column>
x_train1 = final[:891]
feature_scaler = StandardScaler()
x_train1 = feature_scaler.fit_transform(x_train1)
 y_train1 = final[891:]
feature_scaler = StandardScaler()
y_train1 = feature_scaler.fit_transform(y_train1 )
X_train.drop(['sex_unknown', 'width','height'], axis=1, inplace=True)
 test.drop(['width','height'], axis=1, inplace=True )<choose_model_class>
Krange1 = range(1,20)
 scores1 = {}
scores_list1 = []
for k in Krange1:
knn = KNeighborsClassifier(n_neighbors = k)
 knn.fit(x_train1,x_test)
 y_pred = knn.predict(y_train1)
 scores1[k] = metrics.accuracy_score(result['Survived'],y_pred)
 scores_list1.append(metrics.accuracy_score(result['Survived'],y_pred))
plt.plot(Krange,scores_list)
 plt.xlabel("Value of K")
 plt.ylabel("Accuracy" )
xg= xgb.XGBClassifier(
 n_estimators=750,
learning_rate=0.015,
min_child_weight= 59,
max_delta_step= 6,
max_depth= 6,
subsample= 0.751,
colsample_bytree= 0.8595,
gamma= 0,
reg_lambda= 37,
random_state=42,
n_jobs=-1,
)<save_to_csv>
knn = KNeighborsClassifier(n_neighbors = 11)
 Estimator.append(( 'knn',KNeighborsClassifier(n_neighbors = 13)))
 cv = cross_val_score(knn,x_train1,x_test,cv=10)
 Accuracy9 = cv.mean()
Accuracy.append(Accuracy9)
 print(cv)
 print(cv.mean() )
xg.fit(X_train, y_train)
 predictions = xg.predict_proba(test)[:, 1]
meta_df = pd.DataFrame(columns=['image_name', 'target'])
 meta_df['image_name'] = sample['image_name']
meta_df['target'] = predictions
meta_df.to_csv('external_tabular_predicts.csv', header=True, index=False )<save_to_csv>
knn.fit(x_train1,x_test)
 knn.score(y_train1,y_test )
effnet = pd.read_csv('.. /input/blended-effnets-from-previous-notebook/blended_effnets.csv')
 meta = pd.read_csv('./external_tabular_predicts.csv')
 sample['target'] =(
 effnet['target'] * 0.9 +
meta['target'] * 0.1 
)
 sample.to_csv('external_meta_ensembled.csv', header=True, index=False )<install_modules>
model19pred = knn.predict(y_train)
 submission19 = pd.DataFrame(columns = ['PassengerId','Survived'])
 submission19['PassengerId'] = result['PassengerId']
submission19['Survived'] = model19pred
submission19.to_csv('KNN(StdScaler ).csv',index = False )
!pip install -q efficientnet >> /dev/null<import_modules>
models = ['Logistic Regression','SVM Linear Classifier','SVM Polynomial Classifier','Decision Tree','Multinomial NB','Random Forest Classifier','Gradient Boost Classifier','XG Boosting','K-Nearest Neighbors(StdScaler)']
total = list(zip(models,Accuracy))
output2 = pd.DataFrame(total, columns = ['Models after Hyperparameter Tuning','Accuracy after HT'] )
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<load_from_csv>
vot_soft1 = VotingClassifier(estimators = Estimator, voting ='soft')
 vot_soft1.fit(x_train, x_test)
 y_pred = vot_soft1.predict(y_train)
 vot_soft1.score(y_train,y_test)
 modelpred3 = vot_soft1.predict(y_train)
 sub3 = pd.DataFrame(columns = ['PassengerId','Survived'])
 sub3['PassengerId'] = result['PassengerId']
sub3['Survived'] = modelpred3
sub3.to_csv('SoftVoting(HT ).csv',index = False )
<load_from_csv>
vot_soft1.fit(x_train, x_test)
 vot_soft1.score(y_train,y_test )
<load_from_csv>
vot_hard1 = VotingClassifier(estimators = Estimator, voting ='hard')
 vot_hard1.fit(x_train, x_test)
 y_pred = vot_hard1.predict(y_train)
 vot_hard1.score(y_train,y_test)
 modelpred4 = vot_hard1.predict(y_train)
 sub4 = pd.DataFrame(columns = ['PassengerId','Survived'])
 sub4['PassengerId'] = result['PassengerId']
sub4['Survived'] = modelpred4
sub4.to_csv('HardVoting(HT ).csv',index = False )
DEVICE = "TPU"
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('.. /input/siim-isic-melanoma-classification/sample_submission.csv')
 FOLDS = 8
IMG_SIZES = [128, 192, 256, 256, 256, 384, 384, 512]
INC2019 = [1,0,1,1,0,1,0,1]
INC2018 = [0,1,1,0,1,1,1,1]
BATCH_SIZES = [64,64,64,64,64,64,32,32]
EPOCHS = [20, 20, 20, 20, 20, 20, 20, 20]
EFF_NETS = [0,1,2,2,3,3,3,4]
WGTS = [1/FOLDS]*FOLDS
TTA = 15<choose_model_class>
vot_hard1.fit(x_train, x_test)
 vot_hard1.score(y_train,y_test )
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
output = pd.concat([output1,output2],axis = 1)
 output.sort_values(by=['Accuracy after HT'], inplace=True, ascending=False)
 output.head(10 )