Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<categorify>
import os
import joblib
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score, cross_val_predict,RandomizedSearchCV, KFold
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers.experimental import preprocessing
import optuna
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=30, p=0.5),
CoarseDropout(p=0.1),
Cutout(p=0.1),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)<normalization>
train_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 submission = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv' )
class Sq_Ex_Block(nn.Module):
def __init__(self, in_ch, r):
super(Sq_Ex_Block, self ).__init__()
self.se = nn.Sequential(
 GlobalAvgPool() ,
nn.Linear(in_ch, in_ch//r),
nn.ReLU(inplace=True),
nn.Linear(in_ch//r, in_ch),
nn.Sigmoid() 
)
 def forward(self, x):
se_weight = self.se(x ).unsqueeze(-1 ).unsqueeze(-1)
 return x.mul(se_weight)
 class GlobalAvgPool(nn.Module):
def __init__(self):
super(GlobalAvgPool, self ).__init__()
def forward(self, x):
return x.view(*(x.shape[:-2]),-1 ).mean(-1)
 class SE_Net(nn.Module):
def __init__(self,in_channels):
super(SE_Net,self ).__init__()
self.c1 = nn.Conv2d(in_channels=in_channels, out_channels=64,kernel_size=3,stride=1,padding=0)
 self.bn1 = nn.BatchNorm2d(num_features=64,eps=1e-3,momentum=0.01)
 self.c2 = nn.Conv2d(64,64,3,1,0)
 self.bn2 = nn.BatchNorm2d(64,1e-3,0.01)
 self.c3 = nn.Conv2d(64,64,5,1,2)
 self.bn3 = nn.BatchNorm2d(64,1e-3,0.01)
 self.m1 = nn.MaxPool2d(2)
 self.d1 = nn.Dropout(0.4)
 self.c4 = nn.Conv2d(64,128,3,1,0)
 self.bn4 = nn.BatchNorm2d(128,1e-3,0.01)
 self.c5 = nn.Conv2d(128,128,3,1,0)
 self.bn5 = nn.BatchNorm2d(128,1e-3,0.01)
 self.c6 = nn.Conv2d(128,128,5,1,2)
 self.bn6 = nn.BatchNorm2d(128,1e-3,0.01)
 self.m2 = nn.MaxPool2d(2)
 self.d2 = nn.Dropout(0.4)
 self.c7 = nn.Conv2d(128,256,3,1,0)
 self.bn7 = nn.BatchNorm2d(256,1e-3,0.01)
 self.se3 = Sq_Ex_Block(in_ch=256,r=8)
 self.m3 = nn.MaxPool2d(2)
 self.d3 = nn.Dropout(0.4)
 self.fc1 = nn.Linear(256*1*1,256)
 self.bn8 = nn.BatchNorm1d(256,1e-3,0.01)
 self.out = nn.Linear(256,10)
 self.init_linear_weights()
def forward(self,x):
x = self.bn1(F.leaky_relu(self.c1(x),0.1))
x = self.bn2(F.leaky_relu(self.c2(x),0.1))
x = self.bn3(F.leaky_relu(self.c3(x),0.1))
x = self.d1(self.m1(x))
x = self.bn4(F.leaky_relu(self.c4(x),0.1))
x = self.bn5(F.leaky_relu(self.c5(x),0.1))
x = self.bn6(F.leaky_relu(self.c6(x),0.1))
x = self.d2(self.m2(x))
x = self.bn7(F.leaky_relu(self.c7(x),0.1))
x = self.se3(x)
 x = self.d3(self.m3(x))
x = x.view(-1, 256*1*1)
 x = self.bn8(F.leaky_relu(self.fc1(x),0.1))
return self.out(x)
 def init_linear_weights(self):
nn.init.kaiming_normal_(self.fc1.weight, mode='fan_in')
 nn.init.kaiming_normal_(self.out.weight, mode='fan_in' )<load_from_csv>
features = [feature for feature in train_df.columns if feature not in ['id', 'target']]
X_train = train_df[features]
y_train = train_df['target']
X_test = test_df[features]
seed_everything(CFG['seed'])
 test_data = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 test_ds = KMnistDataset(test_data, transforms=get_inference_transforms())
 tst_preds = []
device = torch.device(CFG['device'] )<define_variables>
print('Missing value in train dataset:', sum(train_df.isnull().sum()))
print('Missing value in test dataset:', sum(test_df.isnull().sum()))
model_paths = ['.. /input/eesm5720/SE-Net_fold_0_9',
'.. /input/eesm5720/SE-Net_fold_1_9',
'.. /input/eesm5720/SE-Net_fold_2_9',
'.. /input/eesm5720/SE-Net_fold_3_9',
'.. /input/eesm5720/SE-Net_fold_4_9',]<load_pretrained>
cv = KFold(n_splits=5, shuffle=True, random_state=42 )
for o, model_path in enumerate(model_paths):
model = SE_Net(in_channels=1 ).to(device)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 print("using the model from {}".format(model_path))
model.load_state_dict(torch.load(model_path))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [1/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0 )<prepare_output>
%%time
lin_reg = LinearRegression()
scores = cross_val_score(lin_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 lin_rmse_scores = np.sqrt(-scores)
 print('Linear Regression performance:', lin_rmse_scores )
test_data['label'] = np.argmax(tst_preds, axis=1 )<create_dataframe>
%%time
tree_reg = DecisionTreeRegressor(random_state=42)
 scores = cross_val_score(tree_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 tree_rmse_scores = np.sqrt(-scores)
 print('Decision Tree Regressor performance:', tree_rmse_scores )
test_data = pd.DataFrame(test_data, columns=['id','label'] )<save_to_csv>
%%time
forest_reg = RandomForestRegressor(random_state=42, n_jobs=-1)
 scores = cross_val_score(forest_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 forest_rmse_scores = np.sqrt(-scores)
 print('Random Forest performance:', forest_rmse_scores )
test_data.to_csv('submission.csv', index=False )<install_modules>
%%time
lgbm_reg = LGBMRegressor(random_state=42)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 lgbm_rmse_scores = np.sqrt(-scores)
 print('LGBM performance:', lgbm_rmse_scores )
!pip install efficientnet<import_modules>
%%time
xgb_reg = XGBRegressor(random_state=42)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 xgb_rmse_scores = np.sqrt(-scores)
 print('XGBoost performance:', xgb_rmse_scores )
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from sklearn import metrics
from sklearn.model_selection import train_test_split
from keras.callbacks import ModelCheckpoint, EarlyStopping<define_variables>
%%time
cb_reg = CatBoostRegressor(random_state=42, verbose=False)
 scores = cross_val_score(cb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 cb_rmse_scores = np.sqrt(-scores)
 print('CatBoost performance:', cb_rmse_scores )
GCS_DS_PATH = KaggleDatasets().get_gcs_path('plant-pathology-2020-fgvc7')
 EPOCHS = 200
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IM_Z = 768<define_variables>
%%time
ab_reg = AdaBoostRegressor(random_state=42)
 scores = cross_val_score(ab_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 ab_rmse_scores = np.sqrt(-scores)
 print('AdaBoost performance:', ab_rmse_scores )
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
def build_and_compile_model(norm):
model = keras.Sequential([
norm,
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(1)])
 model.compile(loss='mean_squared_error',
optimizer=tf.keras.optimizers.Adam(0.001))
return model
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
N = len(train_paths)
 print(f"- Độ dài bộ train: {len(train_paths)}")
 print(f"- Độ dài bộ test: {len(test_paths)}")
 N_healthy = train['healthy'].mean() * N
N_multiple_diseases = train['multiple_diseases'].mean() * N
N_rust = train['rust'].mean() * N
N_scab = train['scab'].mean() * N
print(f'Healthy: {N_healthy} mẫu')
 print(f'Multiple diseases: {N_multiple_diseases} mẫu')
 print(f'Rust: {N_rust} mẫu')
 print(f'Scab: {N_scab} mẫu')
 train.describe()<split>
%%time
normalizer = preprocessing.Normalization()
normalizer.adapt(np.array(X_train))
dnn_model = build_and_compile_model(normalizer)
 history = dnn_model.fit(X_train, y_train, validation_split=0.2,
verbose=0, epochs=100 )
train_labels = train.loc[:, 'healthy':].values
print(f"- Train labels:
{train_labels}")
 train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.3, random_state=2020)
 print(f" - Độ dài bộ train: {len(train_paths)}")
 print(f" - Độ dài bộ validation: {len(valid_paths)}" )<normalization>
%%time
lin_reg = LinearRegression()
y_predict = cross_val_predict(lin_reg, X_train, y_train, cv=cv, n_jobs=-1 )
def decode_image(filename, label=None, image_size=(IM_Z, IM_Z)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
%%time
tree_reg = DecisionTreeRegressor(random_state=42)
 y_predict = cross_val_predict(tree_reg, X_train, y_train, cv=cv, n_jobs=-1 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<choose_model_class>
%%time
forest_reg = RandomForestRegressor(random_state=42, n_jobs=-1)
 y_predict = cross_val_predict(forest_reg, X_train, y_train, cv=cv, n_jobs=-1 )
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
ch_p = ModelCheckpoint(filepath="model_ef1.h5", monitor='val_loss', save_weights_only=True, save_best_only=True, verbose=1 )<define_search_space>
%%time
lgbm_reg = LGBMRegressor(random_state=42)
 y_predict = cross_val_predict(lgbm_reg, X_train, y_train, cv=cv, n_jobs=-1 )
lrs = []
lrnfn = build_lrfn()
for epoch in range(0, 100):
lr = lrnfn(epoch)
 lrs.append(lr)
 plt.plot(lrs )<choose_model_class>
%%time
xgb_reg = XGBRegressor(random_state=42)
 y_predict = cross_val_predict(xgb_reg, X_train, y_train, cv=cv, n_jobs=-1 )
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(IM_Z, IM_Z, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()<choose_model_class>
%%time
cb_reg = CatBoostRegressor(random_state=42, verbose=False)
 y_predict = cross_val_predict(cb_reg, X_train, y_train, cv=cv, n_jobs=-1 )
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=0)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
EarlyStop_callback = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True )<train_model>
%%time
ab_reg = AdaBoostRegressor(random_state=42)
 y_predict = cross_val_predict(ab_reg, X_train, y_train, cv=cv, n_jobs=-1 )
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[ch_p, EarlyStop_callback],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)<load_pretrained>
def objective(trial):
params = {
'random_state': 42,
'max_depth': trial.suggest_int('max_depth', 1, 14),
'learning_rate': trial.suggest_float('learning_rate', 0.01, 1.0)
 }
lgbm_reg = LGBMRegressor()
lgbm_reg.set_params(**params)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 rmse = np.sqrt(-scores)
 return np.mean(rmse )
model.load_weights("model_ef1.h5" )<categorify>
study = optuna.create_study(direction = 'minimize')
 study.optimize(objective, n_trials = 1)
 best_params = study.best_trial.params
def dummy_data_augment(image):
return image
data_augments = [dummy_data_augment, tf.image.flip_left_right, tf.image.flip_up_down]<categorify>
%%time
lgbm_reg = LGBMRegressor()
lgbm_reg.set_params(**best_params)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 lgbm_rmse_scores = np.sqrt(-scores)
 print('LGBM performance:', lgbm_rmse_scores )
TTA_test_datasets = [(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.map(d_augment, num_parallel_calls=AUTO)
)for d_augment in data_augments]<predict_on_test>
def objective(trial):
params = {
'random_state': 42,
'max_depth': trial.suggest_int('max_depth', 1, 14),
'eta': trial.suggest_float('eta', 0.01, 1.0),
}
xgb_reg = XGBRegressor()
xgb_reg.set_params(**params)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 rmse = np.sqrt(-scores)
 return np.mean(rmse )
combined_probs_TTAs = [model.predict(dataset)for dataset in TTA_test_datasets]<define_variables>
study = optuna.create_study(direction = 'minimize')
 study.optimize(objective, n_trials = 1)
 best_params = study.best_trial.params
submission_name = 'submission_eff_TTA'
for d_aug in data_augments:
submission_name += "-"
submission_name += d_aug.__name__
submission_name += ".csv"
submission_name<save_to_csv>
%%time
xgb_reg = XGBRegressor()
xgb_reg.set_params(**best_params)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=cv, n_jobs=-1)
 xgb_rmse_scores = np.sqrt(-scores)
 print('XGBoost performance:', xgb_rmse_scores )
sub.loc[:, 'healthy':] = probs_TTA
sub.to_csv(submission_name, index=False)
 sub.head()<set_options>
X_train['below8'] = np.where(y_train < 8, 1, 0 )
<import_modules><EOS> 
%%time
cb_reg = CatBoostRegressor(random_state=42, verbose=False)
 scores = cross_val_score(cb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=5)
 cb_rmse_scores = np.sqrt(-scores)
 print('CatBoost performance:', cb_rmse_scores )