Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<groupby>
warnings.simplefilter("ignore" )
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<define_variables>
train_df = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test_df = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv' )
%%time
N_TRAIN_IMGS = {f'fold_{i}': count_data_items(TRAIN_FNAMES[f'fold_{i}'])
 for i in range(1, NFOLDS+1)}
N_VAL_IMGS = {f'fold_{i}': count_data_items(VAL_FNAMES[f'fold_{i}'])
 for i in range(1, NFOLDS+1)}
N_TEST_IMGS = count_data_items(TEST_FNAMES)
 STEPS_PER_EPOCH = {f'fold_{i}': N_TRAIN_IMGS[f'fold_{i}'] // BATCH_SIZE
for i in range(1, NFOLDS+1)}
print("="*75)
 print(f"The number of unlabeled test image is {N_TEST_IMGS}.It is common for all folds.")
 for i in range(1, NFOLDS+1):
print("="*75)
 print(f"Fold {i}: {N_TRAIN_IMGS[f'fold_{i}']} training and {N_VAL_IMGS[f'fold_{i}']} validation images.")
 print("="*75 )<categorify>
cat_features = [a for a in train_df.columns if a.startswith('cat')]
all_data = pd.concat([train_df[cat_features], test_df[cat_features]], axis=0)
 print(all_data.shape )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image<prepare_x_and_y>
for column in cat_features:
temp = pd.get_dummies(pd.Series(all_data[column]), prefix=column, prefix_sep="_")
 temp_train = temp[0:train_df.shape[0]]
temp_test = temp[train_df.shape[0]:]
train_df = pd.concat([train_df,temp_train],axis=1)
 train_df = train_df.drop([column],axis=1)
 test_df = pd.concat([test_df,temp_test],axis=1)
 test_df = test_df.drop([column],axis=1 )
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"patient_id": tf.io.FixedLenFeature([], tf.string),
"benign_malignant": tf.io.FixedLenFeature([], tf.string),
"age": tf.io.FixedLenFeature([], tf.int64),
"sex_female": tf.io.FixedLenFeature([], tf.int64),
"sex_male": tf.io.FixedLenFeature([], tf.int64),
"sex_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_head/neck": tf.io.FixedLenFeature([], tf.int64),
"site_lower extremity": tf.io.FixedLenFeature([], tf.int64),
"site_oral/genital": tf.io.FixedLenFeature([], tf.int64),
"site_palms/soles": tf.io.FixedLenFeature([], tf.int64),
"site_torso": tf.io.FixedLenFeature([], tf.int64),
"site_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_upper extremity": tf.io.FixedLenFeature([], tf.int64),
"height": tf.io.FixedLenFeature([], tf.int64),
"width": tf.io.FixedLenFeature([], tf.int64),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_scaled": tf.io.FixedLenFeature([], tf.float32),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 data={}
data['image_name']=image_name=tf.cast(example['image_name'], tf.string)
 data['patient_id']=tf.cast(example['patient_id'], tf.string)
 data['age']=tf.cast(example['age'], tf.int32)
 data['sex_female']=tf.cast(example['sex_female'], tf.int32)
 data['sex_male']=tf.cast(example['sex_male'], tf.int32)
 data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)
 data['site_head/neck']=tf.cast(example['site_head/neck'], tf.int32)
 data['site_lower extremity']=tf.cast(example['site_lower extremity'], tf.int32)
 data['site_oral/genital']=tf.cast(example['site_oral/genital'], tf.int32)
 data['site_palms/soles']=tf.cast(example['site_palms/soles'], tf.int32)
 data['site_torso']=tf.cast(example['site_torso'], tf.int32)
 data['site_unknown']=tf.cast(example['site_unknown'], tf.int32)
 data['site_upper extremity']=tf.cast(example['site_upper extremity'], tf.int32)
 data['age_scaled']=tf.cast(example['age_scaled'], tf.float32)
 label=tf.cast(example['target'], tf.int32)
 label_name=tf.cast(example['benign_malignant'], tf.string)
 return image, label, data, label_name<prepare_x_and_y>
features = [a for a in train_df.columns if a.startswith('c')]
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"patient_id": tf.io.FixedLenFeature([], tf.string),
"age": tf.io.FixedLenFeature([], tf.int64),
"sex_female": tf.io.FixedLenFeature([], tf.int64),
"sex_male": tf.io.FixedLenFeature([], tf.int64),
"sex_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_head/neck": tf.io.FixedLenFeature([], tf.int64),
"site_lower extremity": tf.io.FixedLenFeature([], tf.int64),
"site_oral/genital": tf.io.FixedLenFeature([], tf.int64),
"site_palms/soles": tf.io.FixedLenFeature([], tf.int64),
"site_torso": tf.io.FixedLenFeature([], tf.int64),
"site_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_upper extremity": tf.io.FixedLenFeature([], tf.int64),
"height": tf.io.FixedLenFeature([], tf.int64),
"width": tf.io.FixedLenFeature([], tf.int64),
"age_scaled": tf.io.FixedLenFeature([], tf.float32),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 data={}
data['image_name']=image_name=tf.cast(example['image_name'], tf.string)
 data['patient_id']=tf.cast(example['patient_id'], tf.string)
 data['age']=tf.cast(example['age'], tf.int32)
 data['sex_female']=tf.cast(example['sex_female'], tf.int32)
 data['sex_male']=tf.cast(example['sex_male'], tf.int32)
 data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)
 data['site_head/neck']=tf.cast(example['site_head/neck'], tf.int32)
 data['site_lower extremity']=tf.cast(example['site_lower extremity'], tf.int32)
 data['site_oral/genital']=tf.cast(example['site_oral/genital'], tf.int32)
 data['site_palms/soles']=tf.cast(example['site_palms/soles'], tf.int32)
 data['site_torso']=tf.cast(example['site_torso'], tf.int32)
 data['site_unknown']=tf.cast(example['site_unknown'], tf.int32)
 data['site_upper extremity']=tf.cast(example['site_upper extremity'], tf.int32)
 data['age_scaled']=tf.cast(example['age_scaled'], tf.float32)
 return image, data<create_dataframe>
def create_stratified_folds_for_regression(data_df, n_splits=5):

 data_df['kfold'] = -1
data_df = data_df.sample(frac=1 ).reset_index(drop=True)
 num_bins = np.int(np.floor(1 + np.log2(len(data_df))))
print(f"Num bins: {num_bins}")
 data_df.loc[:, "bins"] = pd.cut(pd.to_numeric(data_df['target'], downcast="signed"), bins=num_bins, labels=False)
 kf = model_selection.StratifiedKFold(n_splits=n_splits)
 for f,(t_, v_)in enumerate(kf.split(X=data_df, y=data_df.bins.values)) :
data_df.loc[v_, 'kfold'] = f
data_df = data_df.drop("bins", axis=1)
 return data_df
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled
else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset<train_model>
n_splits = 5
train_df = create_stratified_folds_for_regression(train_df, n_splits )
%%time
training_dataset = load_dataset(TRAIN_FNAMES['fold_2'])
 print("Example of the training data:")
 for image, label, data, label_name in training_dataset.take(1):
print("The image batch size:", image.numpy().shape)
 print("Label:", label.numpy())
 print("Label name:", label_name.numpy())
 print("Age:", data['age'].numpy())
 print("Age(scaled):", data['age_scaled'].numpy() )<train_model>
train_df.kfold.value_counts()
%%time
validation_dataset = load_dataset(VAL_FNAMES['fold_2'])
 print("Examples of the validation data:")
 for image, label, data, label_name in validation_dataset.take(1):
print("The image batch size:", image.numpy().shape)
 print("Label:", label.numpy())
 print("Label name:", label_name.numpy())
 print("Age:", data['age'].numpy())
 print("Age(scaled):", data['age_scaled'].numpy() )<create_dataframe>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FNAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<categorify>
def kfold_splits(n_splits, train_df):

 all_folds = list(range(0, n_splits))
kf_splits = []
for fold in range(0, n_splits):
train_folds = [x for x in all_folds if x != fold]
trn_idx = train_df[train_df.kfold!=fold].index
val_idx = train_df[train_df.kfold==fold].index
kf_splits.append(( fold, trn_idx, val_idx))
return kf_splits
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(databatch):
if len(databatch)==4:
images, labels, _, _ = databatch
numpy_images = images.numpy()
numpy_labels = labels.numpy()
else:
images, _ = databatch
numpy_images = images.numpy()
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192"
if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_image(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2),
color='red' if red else 'black', fontdict={'verticalalignment':'center'},
pad=int(titlesize/1.5)
)
 return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_image(image, title, subplot,
not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()<define_variables>
params = {'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
'feature_pre_filter': False,
'learning_rate': 0.0035,
'lambda_l1': 18.42,
'lambda_l2': 4.02,
'num_leaves': 128,
'min_data_in_leaf': 81,
'sub_feature': 0.5,
'sub_row': 0.8,
'subsample_freq': 10}
training_dataset = training_dataset.batch(20)
 train_batch = iter(training_dataset )<define_variables>
y = train_df['target']
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
random_state = 42
num_round = 15000
for fold, trn_idx, val_idx in kfold_splits(n_splits, train_df):
print(f"fold: {fold}, train len: {len(trn_idx)}, val len: {len(val_idx)}")
 trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=y.iloc[val_idx])
 clf = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 500)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ n_splits
print(f'CV score: {np.round(mean_squared_error(y, oof, squared=False),5)}' )
display_batch_of_images(next(train_batch))<set_options>
submission = pd.DataFrame({"id":test_df.id, "target":predictions})
 submission.to_csv('submission.csv', index=False )