Kaggle competition: Santander Customer Satisfaction
def exponential_lr(epoch,
start_lr = 0.00001, min_lr = 0.00001, max_lr = 0.00005*strategy.num_replicas_in_sync,
rampup_epochs = 5, sustain_epochs = 0,
exp_decay = 0.8):
def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):
if epoch < rampup_epochs:
lr =(( max_lr - start_lr)/
rampup_epochs * epoch + start_lr)
 elif epoch < rampup_epochs + sustain_epochs:
lr = max_lr
else:
lr =(( max_lr - min_lr)*
exp_decay**(epoch - rampup_epochs - sustain_epochs)+
min_lr)
 return lr
return lr(epoch,
start_lr,
min_lr,
max_lr,
rampup_epochs,
sustain_epochs,
exp_decay)
 rng = [i for i in range(30)]
y = [exponential_lr(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<train_model>
%matplotlib inline
filepath="flower1.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 lr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr, verbose=True)
 EPOCHS = 35
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
history = model1.fit(
 ds_train,
validation_data=ds_valid,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[lr_callback, checkpoint],
)<load_pretrained>
df_train = pd.read_csv(".. /input/santander-customer-satisfaction/train.csv")
 df_test = pd.read_csv(".. /input/santander-customer-satisfaction/test.csv" )
model1 = tf.keras.models.load_model('./flower1.h5' )<choose_model_class>
df_train.isnull().sum().sum() , df_test.isnull().sum().sum()
EPOCHS = 50
with strategy.scope() :
pretrained_model = tf.keras.applications.InceptionResNetV2(
 include_top=False,
input_shape=[*IMAGE_SIZE, 3],
weights='imagenet', 
)
 for layer in pretrained_model.layers:
if isinstance(layer, tf.keras.layers.BatchNormalization):
layer.momentum = 0.9
model2 = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'],
)<train_model>
rate_insatisfied = df_train.TARGET.value_counts() [1] / df_train.TARGET.value_counts() [0]
rate_insatisfied * 100
filepath="flower2.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 lr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr, verbose=True)
 EPOCHS = 35
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
history = model2.fit(
 ds_train,
validation_data=ds_valid,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[lr_callback, checkpoint],
)<load_pretrained>
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('TARGET', axis = 1), df_train.TARGET,
train_size = 0.8, stratify = df_train.TARGET,
random_state = 42)
 X_train.shape, y_train.shape[0], X_test.shape, y_test.shape[0]
model2 = tf.keras.models.load_model('./flower2.h5' )<predict_on_test>
X_train_clean = X_train.copy()
X_test_clean = X_test.copy()
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm1_probabilities = model1.predict(images_ds)
 cm2_probabilities = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*cm1_probabilities+(1-alpha)*cm2_probabilities
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
alpha = np.argmax(scores)/100
print('Best alpha: ' + str(alpha))<compute_train_metric>
selector = VarianceThreshold(threshold = 0.01)
 selector.fit(X_train_clean)
 mask_clean = selector.get_support()
X_train_clean = X_train_clean[X_train_clean.columns[mask_clean]]
cm_probabilities = alpha*cm1_probabilities+(1-alpha)*cm2_probabilities
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 labels = range(len(CLASSES))
cmat = confusion_matrix(
 cm_correct_labels,
cm_predictions,
labels=labels, 
)
 cmat =(cmat.T / cmat.sum(axis=1)).T<compute_train_metric>
( len(df_train.columns)- 1)- X_train_clean.shape[1]
score = f1_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 precision = precision_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 recall = recall_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 display_confusion_matrix(cmat, score, precision, recall )<define_variables>
remove = []
cols = X_train_clean.columns
for i in range(len(cols)-1):
column = X_train_clean[cols[i]].values
for j in range(i+1,len(cols)) :
if np.array_equal(column, X_train_clean[cols[j]].values):
remove.append(cols[j])
 X_train_clean.drop(remove, axis = 1, inplace=True )
dataset = get_validation_dataset()
dataset = dataset.unbatch().batch(20)
 batch = iter(dataset )<predict_on_test>
from sklearn.feature_selection import SelectKBest, f_classif, mutual_info_classif
from sklearn.metrics import roc_auc_score as auc
from sklearn.model_selection import cross_val_score
import xgboost as xgb
images, labels = next(batch)
 probabilities1 = model1.predict(images)
 probabilities2 = model2.predict(images)
 probabilities = alpha*probabilities1+(1-alpha)*probabilities2
predictions = np.argmax(probabilities, axis=-1)
 display_batch_of_images(( images, labels), predictions )<predict_on_test>
K_vs_score_fc = []
start = time.time()
for k in range(2, 247, 2):
start = time.time()
selector_fc = SelectKBest(score_func = f_classif, k = k)
 X_train_selected_fc = selector_fc.fit_transform(X_train_clean, y_train)
 clf = xgb.XGBClassifier(seed=42)
 auc_fc = cross_val_score(clf, X_train_selected_fc, y_train, cv = 10, scoring = 'roc_auc')
 K_vs_score_fc.append(auc_fc.mean())
 end = time.time()
print("k = {} - auc_fc = {} - Time = {}s".format(k, auc_fc.mean() , end-start))
end = time.time()
print(end - start)
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities1 = model1.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 probabilities = alpha*probabilities1+(1-alpha)*probabilities2
predictions = np.argmax(probabilities, axis=-1)
 print(predictions )<save_to_csv>
K_vs_score_mic = []
for k in range(2, 247, 2):
start = time.time()
selector_mic = SelectKBest(score_func = mutual_info_classif, k = k)
 X_train_selected_mic = selector_mic.fit_transform(X_train_clean, y_train)
 clf = xgb.XGBClassifier(seed=42)
 auc_mic = cross_val_score(clf, X_train_selected_mic, y_train, cv = 10, scoring = 'roc_auc')
 K_vs_score_mic.append(auc_mic.mean())
 end = time.time()
print("k = {} - auc_mic = {} - Time = {}s".format(k, auc_mic.mean() , end-start))
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt(
 'submission.csv',
np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'],
delimiter=',',
header='id,label',
comments='', 
)
 !head submission.csv<install_modules>
selector_fc = SelectKBest(score_func = f_classif, k = 96)
 selector_fc.fit(X_train_clean, y_train)
 mask_selected = selector_fc.get_support()
selected_col = X_train_clean.columns[mask_selected]
selected_col
!pip install -U efficientnet<import_modules>
X_train_selected = X_train[selected_col]
X_test_selected = X_test[selected_col]
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from keras import backend as K
from efficientnet import keras as efn
from kaggle_datasets import KaggleDatasets
import re, math<import_modules>
from skopt import forest_minimize
tf.__version__<set_options>
def tune_xgbc(params):

 print(params)
 learning_rate = params[0]
n_estimators = params[1]
max_depth = params[2]
min_child_weight = params[3]
gamma = params[4]
subsample = params[5]
colsample_bytree = params[6]
mdl = xgb.XGBClassifier(learning_rate = learning_rate, n_estimators = n_estimators, max_depth = max_depth,
min_child_weight = min_child_weight, gamma = gamma, subsample = subsample,
colsample_bytree = colsample_bytree, seed = 42)
 auc = cross_val_score(mdl, X_train_selected, y_train, cv = 10, scoring = 'roc_auc')
 print(auc.mean())
 return -auc.mean()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
print("tpu")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print("REPLICAS: ", REPLICAS )<define_variables>
space = [(1e-3, 1e-1, 'log-uniform'), 
(100, 2000), 
(1, 10), 
(1, 6.) , 
(0, 0.5), 
(0.5, 1.) , 
(0.5, 1.) ]
result = forest_minimize(tune_xgbc, space, random_state = 42, n_random_starts = 20, n_calls = 25, verbose = 1 )
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 20
IMAGE_SIZE =(512, 512)
 TRAIN_IMG_NUM = 12753
VAL_IMG_NUM = 3712
TEST_IMG_NUM = 7382
STEPS_PER_EPOCHS = TRAIN_IMG_NUM // BATCH_SIZE
STEPS_PER_EPOCHS_FULL =(TRAIN_IMG_NUM + VAL_IMG_NUM)// BATCH_SIZE
AUTO = tf.data.experimental.AUTOTUNE
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.5
}<define_variables>
hyperparameters = ['learning rate', 'n_estimators', 'max_depth', 'min_child_weight', 'gamma', 'subsample',
'colsample_bytree']
for i in range(0, len(result.x)) :
print('{}: {}'.format(hyperparameters[i], result.x[i]))
GCS_DATA_PATH = KaggleDatasets().get_gcs_path()
def get_filenames(path):
return tf.io.gfile.glob(GCS_DATA_PATH + path)
 train_filenames = get_filenames("/tfrecords-jpeg-{0}x{1}/train/*.tfrec".format(*IMAGE_SIZE))
val_filenames = get_filenames("/tfrecords-jpeg-{0}x{1}/val/*.tfrec".format(*IMAGE_SIZE))
test_filenames = get_filenames("/tfrecords-jpeg-{0}x{1}/test/*.tfrec".format(*IMAGE_SIZE))<create_dataframe>
from skopt.plots import plot_convergence
def load_dataset(filenames, ordered=True):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=-1)
 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
return dataset.with_options(ignore_order)
 train_records = load_dataset(train_filenames, False)
 val_records = load_dataset(val_filenames, False)
 test_records = load_dataset(test_filenames, True )<categorify>
clf_optimized = xgb.XGBClassifier(learning_rate = result.x[0], n_estimators = result.x[1], max_depth = result.x[2],
min_child_weight = result.x[3], gamma = result.x[4], subsample = result.x[5],
colsample_bytree = result.x[6], seed = 42 )
def decode_image(raw):

 decoded = tf.io.decode_jpeg(raw)
 image = tf.cast(decoded, tf.float32)/ 255.
 image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image<categorify>
clf_optimized.fit(X_train_selected, y_train )
config = tf.io.FixedLenFeature
feature_train = {"class": config([], tf.int64),
"id" : config([], tf.string),
"image": config([], tf.string),}
feature_test = {"id" : config([], tf.string),
"image": config([], tf.string),}
def read_tfrecord(example_single, features):

 parsed = tf.io.parse_single_example(example_single, features)
 idm = parsed.get("id")
 label = parsed.get("class")
 image = decode_image(parsed.get("image"))
if not features.get("class"):
return image, idm
return image, label
parse_train = partial(read_tfrecord, features=feature_train)
 parse_test = partial(read_tfrecord, features=feature_test )<categorify>
y_predicted = clf_optimized.predict_proba(X_test_selected)[:,1]
auc(y_test, y_predicted )
train_dataset = train_records.map(parse_train, num_parallel_calls=AUTO)
 val_dataset = val_records.map(parse_train, num_parallel_calls=AUTO)
 test_dataset = test_records.map(parse_test )<normalization>
y_predicted_df_test = clf_optimized.predict_proba(df_test[selected_col])[:, 1]
<normalization><EOS> 
sub = pd.Series(y_predicted_df_test, index = df_test['ID'], name = 'TARGET')
 sub.to_csv('submission.csv' )