Kaggle competition: Titanic - Machine Learning from Disaster
%.2f'%y,size=14)
 \ 
<define_variables>
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )
VERBOSE = 0
DISPLAY_PLOT = True
oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
for k in range(M1[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 print('
for k in range(M2[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT+15])
 print('
for k in range(M3[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1+30])
 print('
for k in range(M4[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+30])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
train.drop(labels='Cabin',inplace=True,axis=1)
 test.drop(labels='Cabin',inplace=True,axis=1)
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<save_to_csv>
def check_class(x):
if pd.isnull(x['Age']):
return pmean[x['Pclass']]
return x['Age']
pmean=train.groupby('Pclass' ).mean() ['Age']
train['Age']=train.apply(check_class,axis=1)
 test['Age']=test.apply(check_class,axis=1)
if INFER_TEST:
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()
plt.hist(submission.target,bins=100)
 plt.show()
else:
print('We did not predict test set' )<load_from_csv>
test['Fare']=test['Fare'].fillna(np.mean(test['Fare'])).astype(float)
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
<define_variables>
vtrain=train
vtest=test
models = [ "getting-started-with-tfrecords", "melanoma-efficientnetb6-with-attention-mechanism", "triple-stratified-kfold-with-tfrecords"]
models = ["384-E6-with-2018","512-E6","768-E2","512-E5","effb3-fulldata-upsample","effb2-fulldata-upsample"]
models = ["effb0-fulldata-upsample","effb1-fulldata-upsample","effb2-512-fulldata-upsample","effb3-fulldata-upsample","effb4-fulldata-upsample","effb5-fulldata-upsample"]
models = ["effb1-fulldata-upsample",
"melanoma-efficientnetb6-with-attention-mechanism",
"effb2-fulldata-upsample",
"effb4-fulldata-upsample",
"384-E6-with-2018",
"effb0-fulldata-upsample"]
models = ["384-E6-with-2018","512-E6","768-E2","512-E5","effb2-512-fulldata-upsample",
"effb1-fulldata-upsample"]
for model in models:
dirname = "/kaggle/input/siim3reorg/" + model
_oof = pd.read_csv(os.path.join(dirname, "oof.csv"))
score = metrics.roc_auc_score(_oof['target'], _oof['pred'])
 print(f"{model}: OOF auc:{score:.4}")
 _oof = _oof.rename(columns={"pred":model} ).drop(["target"],axis=1)
 if "fold" in _oof.columns:
_oof = _oof.drop(["fold"],axis=1)
 train = train.merge(_oof, on="image_name")
 _sub = pd.read_csv(os.path.join(dirname, "submission.csv"))
_sub.columns = ["image_name",model]
test = test.merge(_sub, on="image_name")
<feature_engineering>
train=train.loc[:,['Pclass','Survived','Sex','Age','SibSp','Parch','Fare','Embarked']]
test=test.loc[:,['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']]
train["pred_rank"] = 0
train["pred_power"] = 0
train["pred_avg"] = 0
for c in models:
train["pred_rank"] += train[c].rank() / train[c].rank().max()
train["pred_power"] += np.power(train[c],2)/np.power(train[c],2 ).max()
train["pred_avg"] += train [c]/train [c].max()
train["pred_rank"] = train["pred_rank"]/len(models)
 train["pred_power"] = train["pred_power"]/len(models)
 train["pred_avg"] = train["pred_avg"]/len(models)
 score = metrics.roc_auc_score(train['target'], train["pred_avg"])
 print(f'OOF avg_auc:{score}')
 score = metrics.roc_auc_score(train['target'], train["pred_rank"])
 print(f'OOF rank_auc:{score}')
 score = metrics.roc_auc_score(train['target'], train["pred_power"])
 print(f'OOF pow_auc:{score}' )<save_to_csv>
pc=pd.get_dummies(train['Pclass'],drop_first=True,prefix='pclass')
 pctest=pd.get_dummies(test['Pclass'],drop_first=True,prefix='pclass')
 sex=pd.get_dummies(train['Sex'],drop_first=True,prefix='sex')
 sextest=pd.get_dummies(test['Sex'],drop_first=True,prefix='sex')
 em=pd.get_dummies(train['Embarked'],drop_first=True)
 emtest=pd.get_dummies(test['Embarked'],drop_first=True )
test["target"] = 0.0
for c in models:
test["target"] += test[c].rank() / test[c].rank().max()
test["target"] = test["target"]/len(models)
 sub = test[["image_name","target"]]
sub.to_csv("submission_rank.csv",index=False)
 sub.head()<save_to_csv>
def fill(x):
for i in range(len(unique_surnames_train)) :
if unique_surnames_train[i] in x:
return i
extratrain=pd.get_dummies(vtrain["Name"].apply(fill ).replace(np.arange(len(unique_surnames_train)) ,unique_surnames_train),drop_first=True)
 extratest=pd.get_dummies(vtest["Name"].apply(fill ).replace(np.arange(len(unique_surnames_train)) ,unique_surnames_train),drop_first=True)
test["target"] = 0.0
for c in models:
test["target"] += np.power(test[c],2)/np.power(test[c],2 ).max()
test["target"] = test["target"]/len(models)
 sub = test[["image_name","target"]]
sub.to_csv("submission_pow.csv",index=False)
 sub.head()<save_to_csv>
temp=pd.concat([extratest,pd.DataFrame(np.zeros(( test.shape[0],9)) ,columns=['Col.','Don.','Jonkheer.','Lady.','Major.','Mlle.','Mme.','Sir.', 'the'] ).astype('int')],axis=1)
test["target"] = 0.0
for c in models:
test["target"] += test[c]/test[c].max()
test["target"] = test["target"]/len(models)
 sub = test[["image_name","target"]]
sub.to_csv("submission_avg.csv",index=False)
 sub.head()<compute_train_metric>
def hasalpha(x):
for i in x:
if str.isalpha(i):
return i
return 'non'
extr=pd.get_dummies(vtrain['Ticket'].apply(hasalpha)).iloc[:,:-1]
exte=pd.get_dummies(vtest['Ticket'].apply(hasalpha)).iloc[:,:-1].loc[:,extr.columns]
def dim_optimizer(df_oof, features, init_points = 20, n_iter = 30):
pbounds = {'c0':(0.0, 1.0), 'c1':(0.0, 1.0), 'c2':(0.0, 1.0),'c3':(0.0, 1.0),'c4':(0.0, 1.0),'c5':(0.0, 1.0)}
features = features
def dim_opt(df_oof, c0,c1,c2,c3,c4,c5):
x = c0*df_oof[ features[0] ] + c1*df_oof[ features[1]] + c2*df_oof[ features[2]] + c3*df_oof[ features[3]] + c4*df_oof[ features[4]] + c5*df_oof[ features[5]]
return metrics.roc_auc_score(df_oof['target'], x)
 def q(c0, c1,c2,c3,c4,c5):
return dim_opt(df_oof, c0, c1,c2,c3,c4,c5)
 optimizer = BayesianOptimization(
 f=q,
pbounds=pbounds,
random_state=42, 
)
 optimizer.maximize(
 init_points=init_points,
n_iter=n_iter, 
)
 c0 = optimizer.max["params"]["c0"]
c1 = optimizer.max["params"]["c1"]
c2= optimizer.max["params"]["c2"]
c3= optimizer.max["params"]["c3"]
c4= optimizer.max["params"]["c4"]
c5= optimizer.max["params"]["c5"]
t = optimizer.max["target"]
print(f'bo auc:{t}, c0:{c0}, c1:{c1}, c2:{c2},c3:{c3},c4:{c4},c5:{c5}')
 return c0, c1, c2,c3,c4,c5
c0, c1, c2,c3,c4,c5 = dim_optimizer(train, models, init_points = 40, n_iter = 40)
 print(models[0],c0)
 print(models[1],c1)
 print(models[2],c2)
 print(models[3],c3)
 print(models[4],c4)
 print(models[5],c5)
<compute_train_metric>
train=pd.concat([train,pc,sex,em],axis=1 ).drop(['Pclass','Sex','Embarked'],axis=1)
 test=pd.concat([test,pctest,sextest,emtest],axis=1 ).drop(['Pclass','Sex','Embarked'],axis=1 )
def bo_pred(df):
x = c0*df[ models[0] ] + c1*df[ models[1]] + c2*df[ models[2]] + c3*df[ models[3]] + c4*df[ models[4]] + c5*df[ models[5]]
return x
train["pred"] = bo_pred(train)
 score = metrics.roc_auc_score(train['target'], train['pred'])
 print(f"auc bo:{score}")
<save_to_csv>
dftr=pd.read_csv('.. /input/train.csv')
 dfte=pd.read_csv('.. /input/test.csv')
 def app(x):
if pd.notnull(x):
return x[0]
return
xx=pd.get_dummies(dftr['Cabin'].apply(app)).iloc[:,:-2]
xte=pd.get_dummies(dfte['Cabin'].apply(app))
xte['T']=np.zeros(( len(xte),1)).astype('int')
 xte=xte.iloc[:,:-2]
test["target"] = bo_pred(test)
 sub = test[["image_name","target"]]
sub.to_csv("submission_bo.csv",index=False)
 sub.head()<import_modules>
train=pd.concat([train,extratrain,extr,xx],axis=1)
 test=pd.concat([test,extratest,exte,xte],axis=1)
!pip install -q efficientnet 
<load_from_csv>
xtrain=train.iloc[:,1:].values
xtest=test.values
ytrain=train.iloc[:,0].values
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sample = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 train.head()<set_options>
sc_x=StandardScaler()
xtrain=sc_x.fit_transform(xtrain)
 xtest=sc_x.transform(xtest)
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print("REPLICAS: ", strategy.num_replicas_in_sync)
<init_hyperparams>
regressor=LogisticRegression(C=1,solver='saga')
 regressor.fit(xtrain,ytrain )
SEED = 42
FOLDS=3
EFF_NETS = [6]*FOLDS
BATCH_SIZES = [bs * strategy.num_replicas_in_sync for bs in [32]*FOLDS]
IMG_SIZES = [512]*FOLDS
EPOCHS = [10]*FOLDS
DROPOUT = 0.25
LR = 0.00004
WARMUP = 5
CLASS_WEIGHT = {0: train['benign_malignant'].value_counts().malignant/len(train),
1: train['benign_malignant'].value_counts().benign/len(train)}
WGTS = [1/FOLDS]*FOLDS
LABEL_SMOOTHING = 0.05
TTA = 15
INC2018 = [1]*FOLDS
np.random.seed(SEED)
 tf.random.set_seed(SEED )<define_variables>
regressor2=SVC(C=1,gamma=0.01,kernel='rbf')
 regressor2.fit(xtrain,ytrain )
DATASET = {512: '512x512-melanoma-tfrecords-70k-images',
384: 'melanoma-384x384',
192: 'melanoma-192x192'}
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path(DATASET[IMG_SIZES[0]])
 GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
train_filenames = tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')
 test_filenames = tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec' )<init_hyperparams>
regressor3=RandomForestClassifier(criterion='entropy',n_estimators=500)
 regressor3.fit(xtrain,ytrain )
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.000003 * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.3
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<init_hyperparams>
regressor3.score(xtrain,ytrain )
lr_start = 0.000005
lr_max = 0.000003 * BATCH_SIZES[0]
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.3
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
rng = [i for i in range(8 if EPOCHS[0]<8 else EPOCHS[0])]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<normalization>
test['Survived']=regressor.predict(xtest )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=512):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<categorify>
param_svm=[
{
'kernel':['rbf'],
'C':[0.1,0.01,1,5,10],
'gamma':[1,0.1,0.01]
},
{
'kernel':['linear'],
'C':[0.1,0.01,1,5,10],
'gamma':[1,0.1,0.01,5,10]
},
{
'kernel':['sigmoid'],
'C':[0.1,0.01,1,5,10],
'gamma':[1,0.1,0.01]
}
]
param_rf=[
{ 'n_estimators':[10,100,300,600,500], 'criterion':['gini'],'max_depth':[2,5,10,20,50,70,100,150],'max_leaf_nodes':[2,5,10,20,50]},
{ 'n_estimators':[10,100,300,600,500], 'criterion':['entropy'],'max_depth':[2,5,10,20,50,70,100,150],'max_leaf_nodes':[2,5,10,20,50]}
]
param_logistic=[
{'solver':['newton-cg'],
'C':[10,0.1,0.01,1]
},
{'solver':['lbfgs'],
'C':[10,0.1,0.01,1]
},
{'solver':['liblinear'],
'C':[10,0.1,0.01]
},
{'solver':['sag'],'C':[10,0.1,0.01,1]
}
]
gc_svm=GridSearchCV(regressor2,param_grid=param_svm,scoring='accuracy',cv=10 ).fit(xtrain,ytrain)
 gc_logistic=GridSearchCV(regressor,param_grid=param_logistic,scoring='accuracy',cv=10 ).fit(xtrain,ytrain)
 gc_rf=GridSearchCV(regressor3,param_grid=param_rf,scoring='accuracy',cv=10 ).fit(xtrain,ytrain)
 regressor3=gc_rf.best_estimator_
regressor3.fit(xtrain,ytrain)
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*IMG_SIZES[0:2], 3])
 return image
def read_tfrecord(example, labeled, return_imgname=False):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
if return_imgname:
return image, idnum
return image, 0
def load_dataset(filenames, labeled=True, ordered=False, return_imgname=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled,
return_imgname=return_imgname), num_parallel_calls=AUTO)
 , or(image, id)pairs if labeled=False
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def data_augment(image, label=None, seed=SEED):
image = transform(image, IMG_SIZES[0])
 image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 image = tf.image.random_brightness(image, max_delta=0.2)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 if label is None:
return image
else:
return image, label
def plot_transform(num_images):
fig, ax = plt.subplots(nrows=3, ncols=num_images, figsize=(12,5))
x =(load_dataset(train_filenames, labeled=True)
.shuffle(SEED)
.batch(BATCH_SIZES[0],drop_remainder=True)
.prefetch(AUTO)
.unbatch().take(5))
images = []
imgs=[]
for r in range(3):
image,_ = iter(x ).next()
images.append(image)
 for i in range(0,num_images):
image = data_augment(image=images[r])
 imgs.append(image)
 for img in range(len(imgs)) :
ax[img//num_images][img%num_images].imshow(imgs[img])
 ax[img//num_images][img%num_images].axis('off' )<data_type_conversions>
print("SVM")
 print(gc_svm.best_params_)
 print('-----------------------------------------------------------------------------------')
 print('logistic regression')
 print(gc_logistic.best_params_)
 print('-----------------------------------------------------------------------------------')
 print('random forest')
 print(gc_rf.best_params_ )
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 eg_ds =(load_dataset(train_filenames, labeled=True)
.batch(BATCH_SIZES[0],drop_remainder=True)
.prefetch(AUTO)
.unbatch().take(10*6))
show_dataset(64, 10, 6, eg_ds )<normalization>
print("SVM")
 print(gc_svm.best_score_)
 print('-----------------------------------------------------------------------------------')
 print('logistic regression')
 print(gc_logistic.best_score_)
 print('-----------------------------------------------------------------------------------')
 print('random forest')
 print(gc_rf.best_score_ )
plot_transform(7 )<define_variables>
output_train=pd.DataFrame([regressor.predict(xtrain),regressor2.predict(xtrain)] ).apply(lambda x: x.mode() ).iloc[0].values
output=pd.DataFrame([regressor.predict(xtest),regressor2.predict(xtest)] ).apply(lambda x: x.mode() )
VERBOSE = 2
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(test_filenames),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 np.random.seed(SEED)
 np.random.shuffle(files_train); print('
train_dataset =(load_dataset(files_train, labeled=True)
.repeat() 
.map(data_augment, num_parallel_calls=AUTO)
.shuffle(SEED)
.batch(BATCH_SIZES[fold],drop_remainder=True)
.prefetch(AUTO))
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB6(input_shape=(IMG_SIZES[fold],IMG_SIZES[fold], 3),
weights='imagenet',pooling='avg',include_top=False),
Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
metrics=[tf.keras.metrics.AUC(name='auc')])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]))
print('Training...')
 history = model.fit(
 train_dataset,
epochs=EPOCHS[fold],
callbacks=[sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)// BATCH_SIZES[fold],
validation_data=load_dataset(files_valid, labeled=True)
.cache() 
.batch(BATCH_SIZES[fold])
.prefetch(AUTO),
verbose=VERBOSE)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid =(load_dataset(files_valid, labeled=True, ordered=True)
.cache() 
.repeat() 
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
ct_valid = count_data_items(files_valid)
 STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid =(load_dataset(files_valid, labeled=True, ordered=True)
.cache() 
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds =(load_dataset(files_valid, labeled=False, return_imgname=True, ordered=True)
.cache() 
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test =(load_dataset(files_test, labeled=False, ordered=True)
.repeat() 
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
ct_test = count_data_items(files_test)
 STEPS = TTA * ct_test/BATCH_SIZES[fold]/4
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',
label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',
label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.text(x-0.03*xdist,y-0.13*ydist,'max auc
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',
label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',
label='Val Loss',color='
x = np.argmin(history.history['val_loss'])
 y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)
 plt.ylabel('Loss',size=14)
 plt.xticks(ticks=list(range(EPOCHS[fold])) ,labels=list(range(1, EPOCHS[fold]+1)))
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
print(classification_report(ytrain,output_train))
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<find_best_params>
submission=pd.read_csv('.. /input/gender_submission.csv')
 submission['Survived']=output.iloc[0].values
<save_to_csv><EOS> 
submission.to_csv('finaloutput.csv',index=False )