Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<feature_engineering>
!pip install --upgrade xgboost
xgb.__version__
test_folder = '.. /input/cassava-leaf-disease-classification/test_images/'
submission_df = pd.DataFrame(columns={"image_id","label"})
 submission_df["image_id"] = os.listdir(test_folder)
 submission_df["label"] = 0<predict_on_test>
sub = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv")
 data = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 final_test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv" )
submission_df["label"] = run_predictions_over_image_list(submission_df["image_id"], test_folder )<save_to_csv>
print('Training Data')
 print(data.isnull().sum())
 print()
print()
print('Testing Data')
 print(final_test.isnull().sum() )
submission_df.to_csv("submission.csv", index=False )<define_variables>
columns = final_test.columns[1:]
train = data[columns]
target = data['target']
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
x_train, x_test, y_train, y_test =train_test_split(
 train, target, random_state= 2021, test_size = 0.20)
 xgb_initial = xgb.XGBRegressor()
xgb_initial.fit(x_train, y_train)
 initial_preds = xgb_initial.predict(x_test )
from datetime import datetime
from glob import glob
from scipy.ndimage.interpolation import zoom
from scipy.special import softmax
from skimage import io
from sklearn import metrics
from sklearn.metrics import log_loss
from sklearn.metrics import roc_auc_score, log_loss
from sklearn.model_selection import GroupKFold, StratifiedKFold
from torch import nn
from torch.cuda.amp import autocast, GradScaler
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torchvision import transforms
from tqdm import tqdm
import copy
import cv2
import joblib
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pydicom
import random
import sklearn
import time
import timm
import torch
import torchvision
import warnings<init_hyperparams>
mean_squared_error(y_test, initial_preds, squared=False)
CFG = {
'fold_num': 7,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 512,
'epochs': 32,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 8
}<load_from_csv>
def objective(trial, X_data = train, Y_data = target):
x_train, x_test, y_train, y_test = train_test_split(
 X_data, Y_data, random_state= 2021, test_size = 0.20)
 param = {
'tree_method':'gpu_hist',
'predictor': 'gpu_predictor',
'learning_rate': trial.suggest_discrete_uniform('learning_rate',0.01,0.50,0.05),
'colsample_bylevel': trial.suggest_discrete_uniform('colsample_bylevel',0.01,0.91, 0.1),
'colsample_bynode': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'max_depth': trial.suggest_int('max_depth', 1,10),
'subsample': trial.suggest_discrete_uniform('subsample', 0.20,1, 0.05),
'n_estimators': trial.suggest_categorical('n_estimators',[4000,5000,6000,7000]),
'min_child_weight': trial.suggest_int('min_child_weight', 1,401,step=2),
'lambda': trial.suggest_loguniform('lambda', 1e-8, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-8, 10.0),
'random_state': trial.suggest_categorical('random_state',[2000,3000,4000]),
'gamma': trial.suggest_discrete_uniform('gamma',0.01,2.01, 0.1)
 }
model = xgb.XGBRegressor(**param)
 model.fit(x_train,y_train,eval_set=[(x_train,y_train)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(x_test)
 rmse = mean_squared_error(y_test, preds,squared=False)
 return rmse
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials= 100 )
train.label.value_counts()<load_from_csv>
print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params)
 print('Best objective value:', study.best_value)
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<set_options>
best_trial = study.best_trial.params
best_trial['tree_method'] = 'gpu_hist'
best_trial['predictor'] = 'gpu_predictor'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
img = get_img('.. /input/cassava-leaf-disease-classification/train_images/1000015157.jpg')
 plt.imshow(img)
 plt.show()<categorify>
best_trial= {'learning_rate': 0.01,
'colsample_bylevel': 0.6100000000000001,
'colsample_bytree': 0.91,
'max_depth': 10,
'subsample': 0.8,
'min_child_weight': 67,
'lambda': 0.012157425362490908,
'alpha': 7.278941365308569e-08,
'random_state': 3000,
'gamma': 1,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'}
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<set_options>
final_test = xgb.DMatrix(final_test[columns] )
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion,
HueSaturationValue, IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur,
IAAPiecewiseAffine, RandomResizedCrop, IAASharpen, IAAEmboss,
RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,
ShiftScaleRotate, CenterCrop, Resize 
)
 def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(
 shift_limit=0.025,
scale_limit=0.05,
rotate_limit=20,
p=0.5 
),
HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0 
),
ToTensorV2(p=1.0)
 ], p=1.)<choose_model_class>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<define_variables>
NUM_FOLDS=10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 fold_rmse =[]
for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(best_trial, train_df, 2000)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(final_test)
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
rmse_iter = mean_squared_error(temp_oof, val_target, squared=False)
 print(rmse_iter)
 fold_rmse.append(rmse_iter )
model_path = [
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_0_5",
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_1_9",
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_2_9",
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_3_4",
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_4_3",
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_7_9",
".. /input/cassava-10-fold-label-smoothing-02/cassava_model_10_fold_labelsmoothing_0.2_small/tf_efficientnet_b3_ns_fold_8_8",
]<create_dataframe>
sub['target'] = test_preds
sub.to_csv('submission2_post_competition.csv', index=False )
if __name__ == '__main__':
seed_everything(CFG['seed'])
 tst_preds_all_folds = []
for fold in range(CFG['fold_num']):
test = pd.DataFrame()
test['image_id'] = sorted(list(
 os.listdir('.. /input/cassava-leaf-disease-classification/test_images/')
))
test_ds = CassavaDataset(
 test,
'.. /input/cassava-leaf-disease-classification/test_images/',
transforms=get_inference_transforms() ,
output_label=False 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 tst_preds = []
model.load_state_dict(torch.load(model_path[fold]))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [
inference_one_epoch(model, tst_loader, device)
 ]
tst_preds = np.mean(tst_preds, axis=0)
 tst_preds_all_folds.append(tst_preds)
 del model
torch.cuda.empty_cache()<drop_column>
def objective_2(trial, X_data = train, Y_data = target):
x_train, x_test, y_train, y_test = train_test_split(
 X_data, Y_data, random_state= 2021, test_size = 0.20)
 param = {
'tree_method':'gpu_hist',
'predictor': 'gpu_predictor',
'learning_rate': 0.01,
'colsample_bylevel': trial.suggest_discrete_uniform('colsample_bylevel',0.01,0.91, 0.1),
'colsample_bynode': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree',0.01,0.91, 0.1),
'max_depth': trial.suggest_int('max_depth', 1,10),
'subsample': trial.suggest_discrete_uniform('subsample', 0.20,1, 0.05),
'n_estimators': trial.suggest_categorical('n_estimators',[4000,5000,6000,7000]),
'min_child_weight': trial.suggest_int('min_child_weight', 1,401,step=2),
'lambda': trial.suggest_loguniform('lambda', 1e-8, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-8, 10.0),
'random_state': trial.suggest_categorical('random_state',[2000,3000,4000]),
'gamma': trial.suggest_discrete_uniform('gamma',0.01,2.01, 0.1)
 }
model = xgb.XGBRegressor(**param)
 model.fit(x_train,y_train,eval_set=[(x_train,y_train)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(x_test)
 rmse = mean_squared_error(y_test, preds,squared=False)
 return rmse
variable_list = %who_ls
for _ in variable_list:
if _ is not "tst_preds_all_folds":
del globals() [_]
%who_ls<import_modules>
study_2 = optuna.create_study(direction='minimize')
 study_2.optimize(objective_2, n_trials= 100 )
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 warnings.filterwarnings('ignore')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<define_variables>
print('Number of finished trials:', len(study_2.trials))
print('Best trial:', study_2.best_trial.params)
 print('Best objective value:', study_2.best_value)
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassava-resnext/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'<init_hyperparams>
best_trial_2 = study_2.best_trial.params
best_trial_2['tree_method'] = 'gpu_hist'
best_trial_2['predictor'] = 'gpu_predictor'
best_trial_2['learning_rate'] = 0.01
class CFG:
debug=False
num_workers=8
model_name='resnext50_32x4d'
size=512
batch_size=32
seed=2020
target_size=5
target_col='label'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
inference=True
tta=8<load_from_csv>
train_oof = np.zeros(( 300000,))
test_preds_2 = 0
train_oof.shape
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test['filepath'] = test.image_id.apply(lambda x: os.path.join('.. /input/cassava-leaf-disease-classification/test_images', f'{x}')) 
<normalization>
best_trial_2 = {'colsample_bylevel': 0.91,
'colsample_bytree': 0.6100000000000001,
'max_depth': 10,
'subsample': 0.5,
'min_child_weight': 21,
'lambda': 2.4118345076896113e-05,
'alpha': 3.234942680594196e-08,
'random_state': 3000,
'gamma': 1.51,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor',
'learning_rate': 0.01}
def get_transforms(*, data):
if data == 'valid':
return A.Compose([
A.Resize(CFG.size, CFG.size),
A.Transpose(p=0.5),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2()
] )<choose_model_class>
NUM_FOLDS=10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 fold_rmse_2 =[]
for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(best_trial_2, train_df, 2000)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(final_test)
 train_oof[val_ind] = temp_oof
test_preds_2 += temp_test/NUM_FOLDS
rmse_iter = mean_squared_error(temp_oof, val_target, squared=False)
 print(rmse_iter)
 fold_rmse_2.append(rmse_iter )
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x 
<load_pretrained>
sub['target'] = test_preds_2
sub.to_csv('submission3_post_competition.csv', index=False )