Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<import_modules>
import random
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn.mixture import GaussianMixture
from sklearn.metrics import confusion_matrix
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from lightgbm import LGBMClassifier
from xgboost import XGBRegressor
import optuna
from optuna import Trial
from optuna.samplers import TPESampler
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.metrics import accuracy_score,confusion_matrix,classification_report, precision_recall_curve
from IPython.display import display
import gc
import cv2
from tf2_resnets import models<feature_engineering>
df_train = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 df_test = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 df_train.info(verbose=False, memory_usage='deep' )
def _bytes_feature(value):

 if isinstance(value, type(tf.constant(0))):
value = value.numpy()
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _float_feature(value):

 return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
def _int64_feature(value):

 return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def serialize_example(image,image_name):
feature = {
'image': _bytes_feature(image),
'image_name': _bytes_feature(image_name),
}
example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
return example_proto.SerializeToString()<feature_engineering>
train_mode = False
target = 'target'
random_state = 42
x_cols = [c for c in df_train.columns if 'cont' in c]
RESNEXT_ID = 10
N_TFRECORDS = 20
IMAGE_HEIGHT = 600
IMAGE_WIDTH = 800
os.mkdir('test_tfrecords_600')
 test_df = pd.DataFrame(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'),
columns=['image_name'])
 test_df['tfr_group'] = test_df.index%N_TFRECORDS<save_to_csv>
def _evaluate(model, x_cols, df, target=target, n_folds=5):
oof = np.zeros(len(df[target]))
preds_test = np.zeros(len(df_test))
kf = KFold(n_splits=n_folds, random_state=random_state, shuffle=True)
 for idx_train, idx_test in kf.split(df):
x_train = df.loc[idx_train, x_cols].values
y_train = df.loc[idx_train, target].values
x_test = df.loc[idx_test, x_cols].values
y_test = df.loc[idx_test, target].values
model.fit(x_train, y_train)
 preds_train = model.predict(x_test)
 preds_test += model.predict(df_test[x_cols].values)/ n_folds
oof[idx_test] = preds_train
return oof, preds_test
for tfr_group in range(N_TFRECORDS):
df = test_df[test_df.tfr_group==tfr_group]
if df.shape[0]>0:
tfr_filename = 'test_tfrecords_600/cassava_test{}-{}.tfrec'.format(tfr_group,df.shape[0])
 print("Writing",tfr_filename)
 with tf.io.TFRecordWriter(tfr_filename)as writer:
for index,row in tqdm(df.iterrows()):
image_name = row['image_name']
image_path = '.. /input/cassava-leaf-disease-classification/test_images/'+image_name
image = cv2.imread(image_path)
 image = cv2.resize(image,(IMAGE_WIDTH,IMAGE_HEIGHT))
image_shape = image.shape
image = cv2.imencode('.jpg', image,(cv2.IMWRITE_JPEG_QUALITY, 100)) [1].tostring()
image_name = str.encode(image_name)
 sample = serialize_example(image,image_name)
 writer.write(sample)
 image_shape<feature_engineering>
def objective(trial):
param = {
'boosting_type':'gbdt',
'num_leaves':trial.suggest_int('num_leaves', 3, 150),
'max_depth':trial.suggest_int('max_depth', -1, 20),
'learning_rate':trial.suggest_float('learning_rate', 0.001, 0.6),
'n_estimators':trial.suggest_int('n_estimators', 50, 500),
'min_child_weight':trial.suggest_float('min_child_weight', 0.2, 0.6),
'min_child_samples':trial.suggest_int('min_child_samples', 15, 30),
'subsample':trial.suggest_float('subsample', 0.5, 1.0),
'subsample_freq':trial.suggest_int('subsample_freq', 3, 150),
'random_state':random_state,
'lambda_l1':trial.suggest_float('lambda_l1', 0.0, 5.0)
 }
lgbm = LGBMRegressor(**param)
 x_cols = [c for c in df_train.columns if 'cont' in c]
oof, preds_test = _evaluate(lgbm, x_cols, df_train.reset_index(drop=True))
return mean_squared_error(df_train[target].reset_index(drop=True), oof, squared=False)
 if train_mode:
study = optuna.create_study(direction='minimize', sampler=TPESampler())
 study.optimize(objective, n_trials=200)
 print(study.best_value)
 print(study.best_params )
N_TFRECORDS = 20
IMAGE_HEIGHT = 666
IMAGE_WIDTH = 500
os.mkdir('test_tfrecords_500')
 test_df = pd.DataFrame(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'),
columns=['image_name'])
 test_df['tfr_group'] = test_df.index%N_TFRECORDS<save_to_csv>
def objective(trial):
param = {
'iterations':1000,
'verbose':False,
'random_state':random_state,
'loss_function':'RMSE',
'bootstrap_type':'Bernoulli',
'learning_rate':trial.suggest_float('learning_rate', 0.0001, 0.31),
'max_depth':trial.suggest_int('max_depth', 3, 10),
'colsample_bylevel':trial.suggest_float('colsample_bylevel', 0.3, 0.8),
}
cat = CatBoostRegressor(**param)
 x_cols = [c for c in df_train.columns if 'cont' in c]
oof, preds_test = _evaluate(cat, x_cols, df_train, target=target)
 return mean_squared_error(df_train[target], oof, squared=False)
 if train_mode:
study = optuna.create_study(direction='minimize', sampler=TPESampler())
 study.optimize(objective, n_trials=100)
 print(study.best_value)
 print(study.best_params )
for tfr_group in range(N_TFRECORDS):
df = test_df[test_df.tfr_group==tfr_group]
if df.shape[0]>0:
tfr_filename = 'test_tfrecords_500/cassava_test{}-{}.tfrec'.format(tfr_group,df.shape[0])
 print("Writing",tfr_filename)
 with tf.io.TFRecordWriter(tfr_filename)as writer:
for index,row in tqdm(df.iterrows()):
image_name = row['image_name']
image_path = '.. /input/cassava-leaf-disease-classification/test_images/'+image_name
image = cv2.imread(image_path)
 image = cv2.resize(image,(IMAGE_WIDTH,IMAGE_HEIGHT))
image_shape = image.shape
image = cv2.imencode('.jpg', image,(cv2.IMWRITE_JPEG_QUALITY, 100)) [1].tostring()
image_name = str.encode(image_name)
 sample = serialize_example(image,image_name)
 writer.write(sample)
 image_shape<define_variables>
def objective(trial):
param = {
'random_state':random_state,
'objective':'reg:squarederror',
'booster':'gbtree',
'learning_rate':trial.suggest_float('learning_rate', 0.001, 0.1),
'alpha':trial.suggest_float('alpha', 0.001, 0.1),
'colsample_bylevel':trial.suggest_float('colsample_bylevel', 0.05, 0.1),
'colsample_bytree':trial.suggest_float('colsample_bytree', 0.1, 0.9),
'gamma':trial.suggest_float('gamma', 0, 0.5),
'max_depth':trial.suggest_int('max_depth', 3, 18),
'min_child_weight': trial.suggest_float('min_child_weight', 1, 20),
'subsample':trial.suggest_float('subsample', 0.3, 0.7),
}
xgb = XGBRegressor(**param)
 x_cols = [c for c in df_train.columns if 'cont' in c]
oof, preds_test = _evaluate(xgb, x_cols, df_train, target=target)
 return mean_squared_error(df_train[target], oof, squared=False)
 if train_mode:
study = optuna.create_study(direction='minimize', sampler=TPESampler())
 study.optimize(objective, n_trials=100)
 print(study.best_value)
 print(study.best_params )
DEVICE = "GPU"
FOLDS = 5
FOLD_TO_RUN = [0,1,2,3,4]
BATCH_SIZE = 128
EPOCHS = 15
N_WORKERS = 4 
<choose_model_class>
param_lgbm = {'num_leaves': 148, 'max_depth': 19, 'learning_rate': 0.04168752594808129, 'n_estimators': 468, 'min_child_weight': 0.35392113973764505, 'min_child_samples': 29, 'subsample': 0.9348697769228501, 'subsample_freq': 6, 'lambda_l1': 4.639129744838143}
param_cat = {'learning_rate': 0.07468089271003528, 'max_depth': 8, 'colsample_bylevel': 0.7338241468797853}
param_xgb = param_xgb = {'learning_rate': 0.0652222304334701, 'alpha': 0.0036866921576056855, 'colsample_bylevel': 0.09959606060270643, 'colsample_bytree': 0.863554381598069, 'gamma': 0.3959383978062547, 'max_depth': 15, 'min_child_weight': 19.357558021086128, 'subsample': 0.6991638855748524}
lgbm = LGBMRegressor(seed=random_state, **param_lgbm)
 oof_lgbm, preds_lgbm = _evaluate(lgbm, x_cols, df_train, n_folds=5)
 print('hyper lgbm:', mean_squared_error(df_train[target], oof_lgbm, squared=False))
cat = CatBoostRegressor(iterations=1000, verbose=False, random_state=random_state, loss_function='RMSE', bootstrap_type='Bernoulli', **param_cat)
 oof_cat, preds_cat = _evaluate(cat, x_cols, df_train, n_folds=5)
 print('hyper cat:', mean_squared_error(df_train[target], oof_cat, squared=False))
xgb = XGBRegressor(random_state=random_state, **param_xgb)
 oof_xgb, preds_xgb = _evaluate(xgb, x_cols, df_train, n_folds=2)
 print('hyper xgb:', mean_squared_error(df_train[target], oof, squared=False))
print('hyper blend:', mean_squared_error(df_train[target],(( oof_lgbm * 0.4)+(oof_cat * 0.4)+(oof_xgb * 0.2)) , squared=False))
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<sort_values>
df_test['hyper_lgbm'] = preds_lgbm
df_test['hyper_cat'] = preds_cat
df_test['hyper_xgb'] = preds_xgb
df_test['hyper_blend'] =(preds_lgbm * 0.4)+(preds_cat * 0.4)+(preds_xgb * 0.2)
 df_test['target'] = df_test['hyper_blend']
df_test[['id','target']].to_csv('submission_hyper_blend.csv', index=False )
GCS_PATH = '.'
files_test_600 = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test_tfrecords_600/*.tfrec')))
 files_test_500 = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test_tfrecords_500/*.tfrec')))
 print(files_test_600)
 print(files_test_500 )<define_variables>
n_folds = 3
oof_lgbm_seeds = np.zeros(len(df_train))
oof_car_seeds = np.zeros(len(df_train))
preds_lgbm_seeds = np.zeros(len(df_test))
preds_car_seeds = np.zeros(len(df_test))
for g in range(n_folds):
rand_num = random.randint(1, 5000)
 lgbm = LGBMRegressor(seed=rand_num, **param_lgbm)
 oof, preds_lgbm = _evaluate(lgbm, x_cols, df_train, n_folds=n_folds)
 oof_lgbm_seeds +=(oof / n_folds)
 preds_lgbm_seeds +=(preds_lgbm / n_folds)
 cat = CatBoostRegressor(iterations=1000, verbose=False, random_state=rand_num, loss_function='RMSE', bootstrap_type='Bernoulli', **param_cat)
 oof, preds_cat = _evaluate(cat, x_cols, df_train, n_folds=n_folds)
 oof_car_seeds +=(oof / n_folds)
 preds_car_seeds +=(preds_cat / n_folds )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
print('hyper lgbm:', mean_squared_error(df_train[target], oof_lgbm_seeds, squared=False))
print('hyper cat:', mean_squared_error(df_train[target], oof_car_seeds, squared=False))
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<normalization>
df_test['seed_lgbm'] = preds_lgbm_seeds
df_test['seed_cat'] = preds_car_seeds
df_test['seed_blend'] =(preds_lgbm_seeds * 0.4)+(preds_car_seeds * 0.6)
 df_test['target'] = df_test['seed_blend']
df_test[['id','target']].to_csv('submission_seed_blend.csv', index=False )
def read_unlabeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name']
def prepare_image(img, augment=True, tta=None, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 if dim==600:
img = tf.image.crop_to_bounding_box(img, 0, 100, 600, 600)
 else:
img = tf.image.crop_to_bounding_box(img, 83, 0, 500, 500)
 img = tf.image.resize(img, [dim,dim])
 img = tf.cast(img, tf.float32)/ 255.0
if tta!=None:
if tta[-3:]=='_lr':
img = tf.image.flip_left_right(img)
 tta = tta[:-3]
if tta=='rotate90':
img = tf.image.rot90(img,1)
 if tta=='rotate180':
img = tf.image.rot90(img,2)
 if tta=='rotate270':
img = tf.image.rot90(img,3)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n)
 count_data_items(files_test_600),count_data_items(files_test_500 )<create_dataframe>
lgbm_gmm_class = LGBMClassifier(random_state=random_state)
 oof, preds = _evaluate(lgbm_gmm_class, x_cols, df_gmm_train, target='target_gmm')
 print(accuracy_score(df_gmm_train['target_gmm'], oof))
print(confusion_matrix(df_gmm_train['target_gmm'], oof))
def get_dataset(files, augment = False, tta=None, shuffle = False, repeat = False,
batch_size=16, dim=512):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, tta=tta, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
param_lgbm_gmm_class = {'num_leaves': 144, 'max_depth': 10, 'learning_rate': 0.0934759313797003, 'n_estimators': 107, 'min_child_weight': 0.30363944085393396, 'min_child_samples': 17, 'subsample': 0.9463879414095, 'subsample_freq': 80, 'lambda_l1': 4.67131782429971}
lgbm_gmm_class = LGBMClassifier(random_state=random_state, **param_lgbm_gmm_class)
 lgbm_gmm_reg_0 = LGBMRegressor(seed=random_state)
 lgbm_gmm_reg_1 = LGBMRegressor(seed=random_state)
 lgbm_gmm_reg = LGBMRegressor(seed=random_state, **param_lgbm)
 lgbm_cat_reg = CatBoostRegressor(iterations=1000, verbose=False, random_state=random_state, loss_function='RMSE', bootstrap_type='Bernoulli', **param_cat)
 mask_gmm_0 = df_gmm_train['target_gmm'] == 0
mask_gmm_1 = df_gmm_train['target_gmm'] == 1
lgbm_gmm_class.fit(df_gmm_train[x_cols], df_gmm_train['target_gmm'])
 lgbm_gmm_reg_0.fit(df_gmm_train.loc[mask_gmm_0, x_cols].reset_index(drop=True), df_gmm_train.loc[mask_gmm_0, target].reset_index(drop=True))
lgbm_gmm_reg_1.fit(df_gmm_train.loc[mask_gmm_1, x_cols].reset_index(drop=True), df_gmm_train.loc[mask_gmm_1, target].reset_index(drop=True))
lgbm_gmm_reg.fit(df_gmm_train[x_cols], df_gmm_train[target])
 lgbm_cat_reg.fit(df_gmm_train[x_cols], df_gmm_train[target])
 df_gmm_test[['gmm_class_' + str(a)+ '_pred' for a in lgbm_gmm_class.classes_]] = lgbm_gmm_class.predict_proba(df_gmm_test[x_cols])
 df_gmm_test['lgbm'] = lgbm_gmm_reg.predict(df_gmm_test[x_cols])
 df_gmm_test['cat'] = lgbm_cat_reg.predict(df_gmm_test[x_cols])
 df_gmm_test['hyper_blend'] =(df_gmm_test['lgbm'] * 0.5)+(df_gmm_test['cat'] * 0.5)
 df_gmm_test['gmm_reg_0'] = lgbm_gmm_reg_0.predict(df_gmm_test[x_cols])
 df_gmm_test['gmm_reg_1'] = lgbm_gmm_reg_1.predict(df_gmm_test[x_cols] )
<init_hyperparams><EOS> 
print('blend:', mean_squared_error(df_gmm_test[target], df_gmm_test['hyper_blend'], squared=False))
print('super:', mean_squared_error(df_gmm_test[target], df_gmm_test['super_target'], squared=False))