Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv('/kaggle/input/melanomaextendedtabular/external_upsampled_tabular.csv')
 train.head()<categorify>
sns.set(style="darkgrid")
 train_full = pd.read_csv('.. /input/titanic/train.csv', index_col='PassengerId')
 test_full = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId')
 train_NA = train_full.isna().sum()
test_NA = test_full.isna().sum()
pd.concat([train_NA, test_NA], axis=1, sort = False, keys=['Train NA', 'Test NA'] ).transpose()
train['anatom_site_general_challenge'].replace(['anterior torso','lateral torso','posterior torso'],
'torso', inplace=True )<count_missing_values>
null_fare = test_full[test_full.Fare.isnull() ].index[0]
test_full.loc[null_fare, 'Fare'] = 0
train.isnull().sum()<count_missing_values>
y = train_full.Survived
train_X_full = train_full[train_full.columns.drop('Survived')]
combined = pd.concat([train_full, test_full], axis=0 )
train.isnull().sum()<data_type_conversions>
def alone(df):
if('SibSp' in df.columns)and('Parch' in df.columns):
df['Family'] = df['SibSp'] + df['Parch'] + 1
le = LabelEncoder()
df['le_Ticket'] = le.fit_transform(df['Ticket'])
 df['Same_Ticket'] = df.duplicated(['le_Ticket'])
 df['Alone'] = np.where(( df['Family'] > 1)|(df['Same_Ticket']), False, True)
 plt.figure(figsize=(20, 12))
sns.catplot(x="Alone", kind="count", palette="ch:.25", data=df)
 plt.title('Number of passengers travelling alone or not')
 df = df[df.columns.drop('SibSp', errors='ignore')]
df = df[df.columns.drop('Parch', errors='ignore')]
df = df[df.columns.drop('le_Ticket', errors='ignore')]
df = df[df.columns.drop('Same_Ticket', errors='ignore')]
return df
def fillna(df, column):
na_idx = df[df[column].isnull() ].index.tolist()
prob = df[column].value_counts(normalize=True ).sort_index().tolist()
for i in na_idx:
df.iloc[i, df.columns.get_loc(column)] = choices(sorted(df[column].dropna().unique().tolist()), prob)[0]
train.sex.replace('unknown', np.nan, inplace=True)
 fillna(train, 'sex')
 fillna(train, 'anatom_site_general_challenge')
 fillna(test, 'anatom_site_general_challenge')
 train['age_approx'] = train['age_approx'].fillna(round(train['age_approx'].mean()))
print('Train Missing Data Count: %i, Test Missing Data Count: %i'%(train.isnull().sum().sum() , test.isnull().sum().sum()))<categorify>
combined = alone(combined)
 train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
train = pd.get_dummies(train, columns=["anatom_site_general_challenge"], prefix='site',
drop_first=True)
 train.replace({'sex': {'female':0, 'male': 1}}, inplace=True)
 train.head()<categorify>
def fix_cabin(df):
t = df.Cabin.fillna('U')
 df['Cabin'] = t.str.slice(0,1)
 plt.figure(figsize=(20, 15))
sns.catplot(x="Cabin", kind="count", palette="ch:.25", data=df)
 plt.title('Number of passengers per cabin')
test = pd.get_dummies(test, columns=["anatom_site_general_challenge"], prefix='site',
drop_first=True)
 test.replace({'sex': {'female':0, 'male': 1}}, inplace=True)
 test.head()<normalization>
fix_cabin(combined)
 train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
y_train = train['target']
X_train = train.drop(['image_name','height','width','target'],
axis=1)
 X_test = test.drop(['image_name','patient_id'],axis=1)
 scaler = StandardScaler()
X_train_s = X_train.copy()
X_train_s[['age_approx']] = scaler.fit_transform(X_train_s[['age_approx']])
 X_test_s = X_test.copy()
X_test_s[['age_approx']] = scaler.fit_transform(X_test_s[['age_approx']])
 X_train_s.head()<train_on_grid>
fix_embark(combined)
 train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
lr = LogisticRegression(penalty='l2', class_weight='balanced', random_state=SEED)
 nb = GaussianNB()
rf = RandomForestClassifier(n_estimators=1000, max_depth=2, class_weight='balanced',
n_jobs=-1, random_state=SEED)
 estimators = [lr, rf, nb]
cv = StratifiedKFold(5, shuffle=True, random_state=SEED)
 def model_cv(X_train, y_train, estimators, cv):
for est in estimators:
model_name = est.__class__.__name__
cv_results = cross_validate(est, X_train, y_train, cv=cv,
scoring='roc_auc',
return_train_score=True,
n_jobs=-1)
 train_score = cv_results['train_score'].mean()
test_score = cv_results['test_score'].mean()
print('%s - Train Score: %.3f, Val Score: %.3f'%(model_name,train_score,test_score))
model_cv(X_train_s, y_train, estimators, cv )<save_to_csv>
def model_blend(X_train, y_train, X_test, estimators):
mean_prob = 0
for est in estimators:
est.fit(X_train, y_train)
 mean_prob += est.predict_proba(X_test)[:,1]
return mean_prob/len(estimators)
 meta_df = pd.DataFrame(columns=['image_name', 'target'])
 meta_df['image_name'] = sample['image_name']
meta_df['target'] = model_blend(X_train_s, y_train, X_test_s, estimators)
 meta_df.to_csv('meta_pred.csv', header=True, index=False)
 meta_df.head()<save_to_csv>
def extract_title(df):
if 'Name' in df.columns:
df['Title'] = df['Name'].str.split(',', expand=True)[1].str.split('.', expand=True)[0].str.strip()
df = df[df.columns.drop('Name')]
df['Title'] = df['Title'].replace({'Ms': 'Miss', 'Mlle': 'Miss',
'Mme': 'Mrs', 'Lady': 'Mrs', 'the Countess': 'Mrs', 'Dona': 'Mrs',
'Don': 'Other', 'Rev': 'Other', 'Jonkheer': 'Other', 'Dr': 'Other',
'Sir': 'Other', 'Col': 'Other', 'Capt': 'Other', 'Major': 'Other'})
 sns.catplot(kind="count", y="Title", palette="ch:.25", data=df, order=df['Title'].value_counts().index)
 plt.title('Number of passengers per Title')
 return df
submission_effnet_ensemble = pd.read_csv('.. /input/effnet-ensemble/submission_effnet_ensemble.csv')
 submission.target =(submission_effnet_ensemble.target)+(meta_df.target * 0.1)
 submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
temp = train_X_full.copy()
temp['Survived'] = y
print(temp['Title'].unique())
import numpy as np
import pandas as pd
import numpy as np
import pandas as pd
import os 
<load_from_csv>
def extract_ticket(df):
df['Ticket_Letters'] = df['Ticket'].str.replace('\d+', '')
 df.loc[df['Ticket_Letters']=='','Ticket_Letters'] = 'NA'
df.drop(columns=['Ticket'], inplace=True)
 return df
test = pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv')
 first = pd.read_csv('.. /input/output-of-best-public-submission/submission_best.csv')
 second = pd.read_csv('.. /input/output-of-best-public-submission/submission_first.csv')
 third = pd.read_csv('.. /input/output-of-best-public-submission/submission_post_process.csv')
 submission = first.copy()
second.head()<save_to_csv>
combined = extract_ticket(combined)
 train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
train_X_full.head()
arg1 =(2/3)*first['target']
arg2 =(1/6)*second['target']
arg3 =(1/6)*third['target']
submission['target'] = arg1 + arg2 + arg3
submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
!pip install -q efficientnet >> /dev/null<import_modules>
temp = combined.copy()
to_one_hot = ['Sex', 'Embarked', 'Ticket_Letters', 'Cabin', 'Title', 'Alone', 'FareBin']
temp = pd.concat([temp, pd.get_dummies(temp[to_one_hot])], axis=1)
 temp.drop(columns=to_one_hot, axis=1, inplace=True)
 combined = temp
combined.head()
import json
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<define_search_space>
train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
DEVICE = "TPU"
SEED = 27
FOLDS = 5
IMG_SIZES = [512,512,512,512,512]
INC2019 = [1,1,1,1,1]
INC2018 = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [20]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
def convert_cat(df):
le = LabelEncoder()
le_train_X = df.copy()
s = df.dtypes=='object'
cat_features = list(s[s].index)
 for col in cat_features:
le_train_X[col] = le.fit_transform(df[col])
 return le_train_X
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
without_survived = combined.copy()
without_survived.drop(columns=['Survived'], inplace=True)
 without_survived.head()
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
print(GCS_PATH)
 print()
print(GCS_PATH2)
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
missing_age_model = predict_age(without_survived )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
missing_age = without_survived[without_survived.Age.isnull() ]
missing_age = missing_age[missing_age.columns.drop('Age')]
pred_age = missing_age_model.predict(missing_age)
 output_age = pd.DataFrame({'Age': pred_age}, index=missing_age.index)
 output_age.transpose()
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
test = combined.copy()
test = test.combine_first(output_age)
 pd.DataFrame(test.Age ).transpose()
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
combined = test
train_X_full = combined[combined.Survived.notnull() ]
test_full = combined[combined.Survived.isnull() ]
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
y = train_X_full.Survived
train_X_full = train_X_full.drop(columns=['Survived'], axis=1)
 train_X, valid_X, train_y, valid_y = train_test_split(train_X_full, y, random_state=5 )
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=6):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
def scores(results):
key_min = min(results.keys() , key=(lambda k: results[k]))
key_max = max(results.keys() , key=(lambda k: results[k]))
print('Highest score at %d of %.4f' %(key_max, results[key_max]))
print('Lowest score at %d of %.4f' %(key_min, results[key_min]))
return key_max, key_min
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
print('Number of trees: ')
 high, low = scores(results )
fold = 3
VERBOSE = 2
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
with open(f".. /input/kfold-checkpoints/train{fold}.json", "r")as write_file:
files_train = json.load(write_file)
 with open(f".. /input/kfold-checkpoints/valid{fold}.json", "r")as write_file:
files_valid = json.load(write_file)
 files_test = np.load(f".. /input/kfold-checkpoints/test{fold}.npy")
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
best_rf_model = RandomForestClassifier(n_estimators=high,random_state=0 ).fit(train_X, train_y)
 pred_valid1 = best_rf_model.predict(valid_X)
 print('Mean absolute error: \t%.4f' %mean_absolute_error(pred_valid1, valid_y))
print('Accuracy score: \t%.4f' %accuracy_score(valid_y, pred_valid1))
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission{fold}.csv', index=False)
 submission.head()<set_options>
perm = PermutationImportance(best_rf_model, random_state=1 ).fit(valid_X, valid_y)
 eli5.show_weights(perm, feature_names=valid_X.columns.tolist() )
warnings.filterwarnings('ignore' )<load_from_csv>
results1 = {}
for i in range(1, len(train_X.columns)) :
selector = SelectKBest(f_classif, k=i)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols = selected_features.columns[selected_features.var() != 0]
kbest_X = train_X[selected_cols]
kval_X = valid_X[selected_cols]
rf_model = RandomForestClassifier(n_estimators=high,random_state=0 ).fit(kbest_X, train_y)
 pred_valid = rf_model.predict(kval_X)
 results1[i] = accuracy_score(pred_valid, valid_y)
 plt.plot(list(results1.keys()), list(results1.values()))
plt.xlabel('
plt.ylabel('Accuracy score')
 plt.show()
train = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/train.csv")
 test = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/test.csv")
 sample = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv" )<install_modules>
high1, low1 = scores(results1 )
!pip install -q efficientnet<import_modules>
results2 = {}
for i in range(1, len(train_X.columns)) :
selector = SelectKBest(f_classif, k=i)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols = selected_features.columns[selected_features.var() != 0]
kbest_X = train_X[selected_cols]
kval_X = valid_X[selected_cols]
rf_model = RandomForestClassifier(n_estimators=low,random_state=0 ).fit(kbest_X, train_y)
 pred_valid = rf_model.predict(kval_X)
 results2[i] = accuracy_score(pred_valid, valid_y)
 plt.plot(list(results2.keys()), list(results2.values()))
plt.xlabel('
plt.ylabel('Accuracy score')
 plt.show()
import tensorflow as tf
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets<define_variables>
high2, low2 = scores(results2 )
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 GCS_PATH2 = KaggleDatasets().get_gcs_path('malignant-v2-512x512')
 GCS_PATH3 = KaggleDatasets().get_gcs_path('isic2019-512x512')
 filenames_train1 = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 filenames_train2 = tf.io.gfile.glob(GCS_PATH2 + '/train%.2i*.tfrec'%(2*x)for x in range(25))
filenames_train3 = tf.io.gfile.glob(GCS_PATH3 + '/train%.2i*.tfrec'%(2*x)for x in range(25))
filenames_test = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))<randomize_order>
results3 = {}
for i in range(1, len(train_X.columns)) :
selector = SelectKBest(f_classif, k=i)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols = selected_features.columns[selected_features.var() != 0]
kbest_X = train_X[selected_cols]
kval_X = valid_X[selected_cols]
log_model = LogisticRegression(random_state=0 ).fit(kbest_X, train_y)
 pred_valid = log_model.predict(kval_X)
 results3[i] = accuracy_score(pred_valid, valid_y)
 plt.plot(list(results3.keys()), list(results3.values()))
plt.xlabel('
plt.ylabel('Accuracy score')
 plt.show()
filenames_train = np.array(filenames_train1+filenames_train2+filenames_train3)
 np.random.shuffle(filenames_train)
 np.random.shuffle(filenames_train )<feature_engineering>
high3, low3 = scores(results3 )
AUTO = tf.data.experimental.AUTOTUNE<set_options>
xg_model = xgb.XGBClassifier()
results4 = {}
for i in range(1, len(train_X.columns)) :
selector = SelectKBest(f_classif, k=i)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols = selected_features.columns[selected_features.var() != 0]
kbest_X = train_X[selected_cols]
kval_X = valid_X[selected_cols]
xg_model = xgb.XGBClassifier().fit(kbest_X, train_y)
 pred_valid = xg_model.predict(kval_X)
 results4[i] = accuracy_score(pred_valid, valid_y)
 plt.plot(list(results4.keys()), list(results4.values()))
plt.xlabel('
plt.ylabel('Accuracy score')
 plt.show()
cfg = dict(
 batch_size=32,
img_size=512,
lr_start=0.000005,
lr_max=0.00000125,
lr_min=0.000001,
lr_rampup=5,
lr_sustain=0,
lr_decay=0.8,
epochs=12,
transform_prob=1.0,
rot=180.0,
shr=2.0,
hzoom=8.0,
wzoom=8.0,
hshift=8.0,
wshift=8.0,
optimizer='adam',
label_smooth_fac=0.05,
tta_steps=20
)<normalization>
high4, low4 = scores(results4 )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1], dtype='float32')
 zero = tf.constant([0], dtype='float32')
 rotation_matrix = tf.reshape(
 tf.concat([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0),
[3, 3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0),
[3, 3])
 zoom_matrix = tf.reshape(
 tf.concat([
one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero,
zero, one
],axis=0), [3, 3])
 shift_matrix = tf.reshape(
 tf.concat(
 [one, zero, height_shift, zero, one, width_shift, zero, zero, one],
axis=0), [3, 3])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix)) 
<categorify>
feature_cols = train_X.columns
selector = SelectKBest(f_classif, k=high1)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols = selected_features.columns[selected_features.var() != 0]
kbest_X = train_X[selected_cols]
kval_X = valid_X[selected_cols]
best_k_model = RandomForestClassifier(n_estimators=high,random_state=0 ).fit(kbest_X, train_y)
 pred_valid1 = best_k_model.predict(kval_X)
 print('Mean absolute error: \t%.4f' %mean_absolute_error(pred_valid1, valid_y))
print('Accuracy score: \t%.4f' %accuracy_score(valid_y, pred_valid1))
def transform(image, cfg):
DIM = cfg['img_size']
XDIM = DIM % 2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift)
 x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)
 y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])
 z = tf.ones([DIM * DIM], dtype='int32')
 idx = tf.stack([x, y, z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)
 idx3 = tf.stack([DIM // 2 - idx2[0, ], DIM // 2 - 1 + idx2[1, ]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d, [DIM, DIM, 3] )<prepare_x_and_y>
feature_cols = train_X.columns
selector = SelectKBest(f_classif, k=high4)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols2 = selected_features.columns[selected_features.var() != 0]
kbest_X = train_X[selected_cols2]
kval_X = valid_X[selected_cols2]
xg_best_model = xgb.XGBClassifier().fit(kbest_X, train_y)
 pred_valid1 = xg_best_model.predict(kval_X)
 print('Mean absolute error: \t%.4f' %mean_absolute_error(pred_valid1, valid_y))
print('Accuracy score: \t%.4f' %accuracy_score(valid_y, pred_valid1))
def dropout(image, DIM=512, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image<normalization>
cv_results = {}
for i in range(2, 10):
cv_score = cross_val_score(xg_best_model, cv_X, cv_y, cv=i)
 cv_results[i] = cv_score.mean()
def prepare_image(img, cfg=None,droprate=0.5,dropct=8,dropsize=0.2):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['img_size'], cfg['img_size']],
antialias=True)
 img = tf.cast(img, tf.float32)/ 255.0
if cfg['transform_prob'] > tf.random.uniform([1], minval=0, maxval=1):
img = transform(img, cfg)
 if(tf.random.uniform([1], minval=0, maxval=1)> 0.5)&(droprate!=0)&(dropct!=0)&(dropsize!=0):
img = dropout(img, DIM=512, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 return img<prepare_x_and_y>
cv_high, cv_low = scores(cv_results )
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 return example['image'], example['target'] 
<prepare_x_and_y>
cv_score = cross_val_score(best_k_model, cv_X, cv_y, cv=cv_high)
 print('Mean cross-validation score: %.2f' %(cv_score.mean() *100))
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 return example['image'], example['image_name'] 
<count_values>
feature_cols = train_X.columns
selector = SelectKBest(f_classif, k=high3)
 X_new = selector.fit_transform(train_X[feature_cols], train_y)
 selected_features = pd.DataFrame(selector.inverse_transform(X_new),
index=train_X.index,
columns=feature_cols)
 selected_cols1 = selected_features.columns[selected_features.var() != 0]
kbest_X1 = train_X[selected_cols1]
kval_X1 = valid_X[selected_cols1]
best_k_model1 = LogisticRegression(random_state=0 ).fit(kbest_X1, train_y)
 pred_valid1 = best_k_model1.predict(kval_X1)
 print('Mean absolute error: \t%.4f' %mean_absolute_error(pred_valid1, valid_y))
print('Accuracy score: \t%.4f' %accuracy_score(valid_y, pred_valid1))
cv_X = pd.concat([train_X, valid_X])
 cv_y = pd.concat([train_y, valid_y])
 cv_X.head()
def count_data_items(filenames):
n = [
int(re.compile(r'-([0-9]*)\.' ).search(filename ).group(1))
for filename in filenames
]
return np.sum(n )<create_dataframe>
cv_high, cv_low = scores(cv_results )
def getTrainDataset(files, cfg):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.repeat()
ds = ds.shuffle(2048)
 ds = ds.map(lambda img, label: 
(prepare_image(img, cfg=cfg), label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)
 ds = ds.prefetch(AUTO)
 return ds 
<create_dataframe>
cv_score = cross_val_score(best_k_model1, cv_X, cv_y, cv=cv_high)
 print('Mean cross-validation score: %.2f' %(cv_score.mean() *100))
def getTestDataset(files, cfg, augment=False, repeat=False):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, idnum: 
(prepare_image(img, cfg=cfg), idnum),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
test_full.drop(columns=['Survived'], axis=0, inplace=True )
def getLearnRateCallback(cfg):
lr_start = cfg['lr_start']
lr_max = cfg['lr_max'] * strategy.num_replicas_in_sync * cfg['batch_size']
lr_min = cfg['lr_min']
lr_rampup = cfg['lr_rampup']
lr_sustain = cfg['lr_sustain']
lr_decay = cfg['lr_decay']
def lrfn(epoch):
if epoch < lr_rampup:
lr =(lr_max - lr_start)/ lr_rampup * epoch + lr_start
elif epoch < lr_rampup + lr_sustain:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_rampup -
lr_sustain)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
ktest_X = test_full[selected_cols]
selected_cols
with strategy.scope() :
model_input = tf.keras.Input(shape=(cfg['img_size'], cfg['img_size'], 3),
name='img_input')
 dummy = tf.keras.layers.Lambda(lambda x: x )(model_input)
 outputs = []
x = efn.EfficientNetB3(include_top=False,
weights='imagenet',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 x = efn.EfficientNetB4(include_top=False,
weights='imagenet',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 x = efn.EfficientNetB5(include_top=False,
weights='imagenet',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.compile(optimizer=cfg['optimizer'],
loss=[
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac']),
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac']),
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac'])
 ],
metrics=[tf.keras.metrics.AUC(name='auc')])
<create_dataframe>
pred = best_k_model.predict(ktest_X ).astype('int')
 pred
ds_train = getTrainDataset(filenames_train, cfg ).map(lambda img, label:(img,(label, label, label)))
 stepsTrain = count_data_items(filenames_train)/(cfg['batch_size'] * strategy.num_replicas_in_sync )<train_model>
output = pd.DataFrame({'PassengerId': test_full.index,
'Survived': pred})
 output.to_csv('survived.csv', index=False )
history = model.fit(ds_train,
validation_data = None,
verbose=1,
steps_per_epoch = stepsTrain,
validation_steps = 0,
epochs=14,
callbacks=callbacks )<create_dataframe>
ktest_X1 = test_full[selected_cols1]
pred1 = best_k_model1.predict(ktest_X1 ).astype('int')
 output = pd.DataFrame({'PassengerId': test_full.index,
'Survived': pred1})
 output.to_csv('survived1.csv', index=False )
<predict_on_test><EOS> 
ktest_X2 = test_full[selected_cols2]
pred2 = xg_best_model.predict(ktest_X2 ).astype('int')
 output = pd.DataFrame({'PassengerId': test_full.index,
'Survived': pred2})
 output.to_csv('survived2.csv', index=False )