Kaggle competition: Tabular Playground Series - Feb 2021
del training_dataset, train_batch
gc.collect()<init_hyperparams>
import lightgbm as lgb
import optuna.integration.lightgbm as oplgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import category_encoders as ce
import seaborn as sns
LR_START = 0.000005
LR_MAX = 0.00000725 * strategy.num_replicas_in_sync
LR_MIN = 0.000005
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 4
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<prepare_x_and_y>
df_train = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/train.csv")
 df_test = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/test.csv")
 df_sample = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/sample_submission.csv" )
train['target'].values
class_weights = class_weight.compute_class_weight(class_weight='balanced',
classes=np.unique(train['target'].values),
y=train['target'].values, 
)
 class_weights = {i : class_weights[i] for i in range(len(class_weights)) }
print(class_weights )<define_variables>
train_id = df_train["id"]
test_id = df_test["id"]
df_train.drop("id", axis=1, inplace=True)
 df_test.drop("id", axis=1, inplace=True )
tab_feats=['age_scaled',
'sex_female',
'sex_male',
'sex_unknown',
'site_head/neck',
'site_lower extremity',
'site_oral/genital',
'site_palms/soles',
'site_torso',
'site_unknown',
'site_upper extremity'
]
N_TAB_FEATS=len(tab_feats)
 print(f"The number of tabular features is {N_TAB_FEATS}." )<choose_model_class>
cat_features = [f"cat{i}" for i in range(9 + 1)]
%time
def get_model() :
with strategy.scope() :
pretrained_model = efn.EfficientNetB3(input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 pretrained_model.trainable = True
inp1 = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name='inp1')
 inp2 = tf.keras.layers.Input(shape=(N_TAB_FEATS), name='inp2')
 x=pretrained_model(inp1)
 x=tf.keras.layers.GlobalAveragePooling2D()(x)
 x=tf.keras.layers.Dense(512,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(256,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(128,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 y=tf.keras.layers.Dense(100,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(inp2)
 concat=tf.keras.layers.concatenate([y, x])
 output = tf.keras.layers.Dense(1, activation='sigmoid', name='output' )(concat)
 model = tf.keras.models.Model(inputs=[inp1,inp2], outputs=[output])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=[tf.keras.metrics.AUC() ], 
)
 return model<set_options>
ordinal_encoder = ce.ordinal.OrdinalEncoder()
ordinal_encoder.fit(pd.concat([df_train[cat_features], df_test[cat_features]], axis=0))
train_or = ordinal_encoder.transform(df_train[cat_features])
 test_or = ordinal_encoder.transform(df_test[cat_features])
 train_or.columns = [f"OR_{col}" for col in train_or]
test_or.columns = [f"OR_{col}" for col in test_or]
del model
gc.collect()<define_variables>
numerical_features = [f"cont{i}" for i in range(13 + 1)]
if colab:
SAVE_FOLDER=NAME
if tstamp:
time_zone = pytz.timezone('America/Chicago')
 current_datetime = datetime.now(time_zone)
 ts=current_datetime.strftime("%m%d%H%M%S")
 SAVE_FOLDER+='_'+ts
SAVE_FOLDER=PATH/SAVE_FOLDER
if not os.path.exists(SAVE_FOLDER):
os.mkdir(SAVE_FOLDER)
 else:
SAVE_FOLDER=Path('/kaggle/working' )<categorify>
train_x = pd.concat([
df_train[numerical_features],
train_or
], axis=1 )
def setup_input(image, label, data, label_name):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}, label<feature_engineering>
test_x = pd.concat([
df_test[numerical_features],
test_or
], axis=1 )
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 return data, label<categorify>
train_y = df_train["target"]
def get_training_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
oplgb_train_data = oplgb.Dataset(train_x, train_y )
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset<categorify>
oplgb_params = {
"objective": "regression",
"metric": "root_mean_squared_error",
"verbosity": -1,
"learning_rate": 0.01
}
def setup_test_image(image, data):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}
def setup_test_name(image, data):
return data['image_name']
def get_test_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def average_predictions(X, fn):
y_probas=[]
for idx in range(NBEST):
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 gc.collect()
print(f"Predicting: fold {fn}, model {idx+1} out of {NBEST}...")
 with strategy.scope() :
path_best_model=f'best_model_fold_{fn}_{idx}.hdf5'
TRAINED_MODEL_FOLDER=Path('.. /input/effb3512/EffNB3_512_0702063710')
 model=tf.keras.models.load_model(TRAINED_MODEL_FOLDER/path_best_model)
 y=model.predict(X)
 y = rankdata(y)/len(y)
 y_probas.append(y)
 y_probas=np.average(y_probas, axis=0)
 return y_probas<prepare_x_and_y>
folds = KFold(n_splits=5, shuffle=True, random_state=2021 )
%%time
preds = pd.DataFrame({'image_name': np.zeros(len(test)) , 'target': np.zeros(len(test)) })
 test_ds = load_dataset(TEST_FNAMES, labeled=False, ordered=True)
 test_images_ds = test_ds.map(setup_test_image, num_parallel_calls=AUTO)
 test_images_ds = get_test_dataset(test_images_ds)
 test_ds = get_test_dataset(test_ds)
 test_ids_ds = test_ds.map(setup_test_name, num_parallel_calls=AUTO ).unbatch()
preds['image_name'] = next(iter(test_ids_ds.batch(N_TEST_IMGS)) ).numpy().astype('U')
 preds['target'] = np.average([average_predictions(test_images_ds, fn)for fn in range(1, NFOLDS+1)], axis = 0 )<merge>
tuner_cv = oplgb.LightGBMTunerCV(oplgb_params, oplgb_train_data, num_boost_round=1000, early_stopping_rounds=100, folds=folds, verbose_eval=100, time_budget=21600)
 tuner_cv.run()
del sub['target']
sub = sub.merge(preds, on='image_name')
 sub.head()<count_missing_values>
tuner_cv.best_params
print(f"The lengths of the submission file and `test` are {len(sub)} and {len(test)}, respectively.")
 print(f"The number of NA's in the submission file is {sub.isna().sum().sum() }." )<save_to_csv>
class FoldsAverageLGBM:
def __init__(self, folds):
self.folds = folds
self.models = []
def fit(self, lgb_params, train_x, train_y):
oof_preds = np.zeros_like(train_y)
 self.train_x = train_x
self.train_y = train_y.values
for tr_idx, va_idx in tqdm(folds.split(train_x)) :
tr_x, va_x = self.train_x.iloc[tr_idx], self.train_x.iloc[va_idx]
tr_y, va_y = self.train_y[tr_idx], self.train_y[va_idx]
lgb_train_dataset = lgb.Dataset(tr_x, tr_y)
 lgb_valid_dataset = lgb.Dataset(va_x, va_y)
 model = lgb.train(lgb_params, lgb_train_dataset, valid_sets=[lgb_valid_dataset], verbose_eval=100)
 self.models.append(model)
 oof_pred = model.predict(va_x)
 oof_preds[va_idx] = oof_pred
self.oof_preds = oof_preds
def predict(self, test_x):
preds = []
for model in tqdm(self.models):
pred = model.predict(test_x)
 preds.append(pred)
 preds = np.mean(preds, axis=0)
 return preds
def get_feature_importance(self, importance_type="gain"):
feature_names = self.models[0].feature_name()
feature_importances_list = [model.feature_importance(importance_type)for model in self.models]
out_df = pd.DataFrame()
for i, name in enumerate(feature_names):
out_df[name] = [v[i] for v in feature_importances_list]
return out_df
if colab:
OUT_FOLDER=SAVE_FOLDER
else:
OUT_FOLDER=Path('')
 sub.to_csv(OUT_FOLDER/'submission.csv', index=False )<import_modules>
lgb_params = dict(tuner_cv.best_params)
 lgb_params["learning_rate"] = 0.005
lgb_params["early_stopping_round"] = 200
lgb_params["num_iterations"] = 20000
import numpy as np
import pandas as pd
import os<define_variables>
folds_average_lgbm = FoldsAverageLGBM(folds )
sub_path = ".. /input/ensemble-melanoma"
all_files = os.listdir(sub_path)
 all_files = [f for f in all_files if 'seresnext50 mean tta 0.9252.csv' not in f and 'b6 2019 mean 0.8666.csv' not in f and 'cpu densenet121 0.8845.csv' not in f]
all_files += ['B3-B6 80 82 size 512.csv', 'triple-stratified-kfold-with-tfrecords 0.9426.csv']
all_files<load_from_csv>
folds_average_lgbm.fit(lgb_params, train_x, train_y )
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "target" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<save_to_csv>
np.sqrt(mean_squared_error(df_train.target, folds_average_lgbm.oof_preds))
concat_sub['target'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub[['image_name', 'target']].to_csv('submission_mean.csv', index=False, float_format='%.6f' )<define_variables>
y_pred = folds_average_lgbm.predict(test_x )
<install_modules><EOS> 
sub = df_sample.copy()
sub["target"] = y_pred
sub.to_csv("submission_optuna_lgbm_or_v1.csv", index=False)
 sub.head()