Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<import_modules>
%matplotlib inline
import random<save_to_csv>
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))
submission.to_csv('predictions.csv')
 submission<set_options>
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = {}):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(X_train, label=y_train, group_id=[0] * len(X_train))
self.val_data = Pool(X_val, label=y_val, group_id=[0] * len(X_val))
self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError
K.clear_session()<set_options>
ID_COL = "id"
FEAT_COLS = [f"cont{i}" for i in range(1, 15)]
TGT_COL = "target"
N_SPLITS = 10
RANDOM_SEED_LIST = [
1003,2021
]
MODEL_PARAMS = {
"lgb": {
'seed': RANDOM_SEED_LIST[1],
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
'feature_pre_filter': False,
'lambda_l1': 6.540486456085813,
'lambda_l2': 0.01548480538099245,
'num_leaves': 256,
'feature_fraction': 0.52,
'bagging_fraction': 0.6161835249194311,
'bagging_freq': 7,
'min_child_samples': 20,
'learning_rate': 0.005,
"device": "gpu",
"gpu_device_id": 0,
},
"xgb": {
"objective": "reg:squarederror",
"learning_rate": 0.005,
"seed": RANDOM_SEED_LIST[0],
'alpha': 0.01563,
'base_score': 0.5,
'booster': 'gbtree',
'colsample_bylevel': 1,
'colsample_bynode': 1,
'colsample_bytree': 0.5,
'gamma': 0,
'importance_type': 'gain',
'interaction_constraints': '',
'max_delta_step': 0,
'max_depth': 15,
'metric_period': 100,
'min_child_weight': 257,
'n_jobs': 0,
'num_parallel_tree': 1,
'reg_alpha': 0.0156299993,
'reg_lambda': 0.003,
'scale_pos_weight': 1,
'silent': 1,
'subsample': 0.7,
'tree_method': 'exact',
'validate_parameters': 1,
'verbosity': None,
'tree_method': 'gpu_hist',
'gpu_id': 0,
},
"cat": {
'loss_function': 'RMSE',
'learning_rate': 0.01,
'max_depth': 7,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
'num_boost_round': 20000
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 200,
"verbose_eval": 100,
},
"xgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 200,
"verbose_eval": 100,
},
"cat": {'early_stopping_rounds': 200, 'verbose_eval': 100}
}
device = cuda.get_current_device()
device.reset()<define_variables>
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()
package_paths = [
'.. /input/pytorch-image-models/pytorch-image-models-master',
'.. /input/image-fmix/FMix-master'
]
for pth in package_paths:
sys.path.append(pth)
<import_modules>
train_feat = pd.concat([
train_feat, train[FEAT_COLS]], axis=1)
 test_feat = pd.concat([
test_feat, test[FEAT_COLS]], axis=1)
 use_feat_cols.extend(FEAT_COLS )
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import glob
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<import_modules>
def run_train_and_inference(X, X_test, y, use_model, model_params, train_params, seed_list, n_splits):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params)
 fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances
import os
import pandas as pd
import albumentations as albu
import matplotlib.pyplot as plt
import json
import seaborn as sns
import cv2
import albumentations as albu
import numpy as np<import_modules>
X = train_feat[use_feat_cols]
X_test = test_feat[use_feat_cols]
y = train[TGT_COL].values
print(f"train_feat: {X.shape}, test_feat: {X_test.shape}" )
import torch
import torch.nn as nn
import torchvision.models as models
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from torch.optim.lr_scheduler import ReduceLROnPlateau
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split
from albumentations.pytorch import ToTensorV2
import time
import datetime
import copy<init_hyperparams>
oof_pred_lgb, test_pred_lgb, score_lgb, feat_imps_lgb = run_train_and_inference(
 X, X_test, y, "lgb", MODEL_PARAMS["lgb"], TRAIN_PARAMS["lgb"], RANDOM_SEED_LIST, N_SPLITS )
CFG = {
'fold_num': 10,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 384,
'epochs': 32,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 4,
'used_epochs': [8],
'weights': [1,1,1,1,1]
}<set_options>
score_lgb.loc[score_lgb.fold == "oof"]
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb<train_on_grid>
oof_pred_xgb, test_pred_xgb, score_xgb, feat_imps_xgb = run_train_and_inference(
 X, X_test, y, "xgb", MODEL_PARAMS["xgb"], TRAIN_PARAMS["xgb"], RANDOM_SEED_LIST, N_SPLITS )
class CassavaDataset(Dataset):
def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):
self.df=df
self.imfolder=imfolder
self.train=train
self.transforms=transforms
def __getitem__(self,index):
im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])
 x=cv2.imread(im_path,cv2.IMREAD_COLOR)
 x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)
 if(self.transforms):
x=self.transforms(image=x)['image']
if(self.train):
y=self.df.iloc[index]['label']
return x,y
else:
return x
def __len__(self):
return len(self.df )<choose_model_class>
score_xgb.loc[score_xgb.fold == "oof"]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 class CustomDeiT(nn.Module):
def __init__(self, model_name='model_name', pretrained=False):
super().__init__()
self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=0)
 n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 print(self.model)
 n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
class CustomViT(nn.Module):
def __init__(self, model_name='', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features,5)
 def forward(self, x):
x = self.model(x)
 return x
model = CustomViT(model_name='vit_base_patch16_384', pretrained=False ).to(device)
 device = torch.device(CFG['device'])
 model.to(device)
 model.eval()<categorify>
oof_pred_avg = oof_pred_lgb*0.6 + oof_pred_xgb*0.4
oof_score_avg = mean_squared_error(y, oof_pred_avg, squared=False)
 print(f"oof score avg: {oof_score_avg:.5f}" )
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<normalization>
test_pred_avg = test_pred_lgb*0.6 + test_pred_xgb*0.4
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,RandomCrop, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_inference_transforms(CFG):
return Compose([
RandomCrop(512,512),
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.) 
<prepare_output>
sub = smpl_sub.copy()
sub[TGT_COL] = test_pred_avg
sub.to_csv("submission.csv", index=False)
 sub.head()