Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<choose_model_class>
!pip3 install seaborn==0.11.0
!pip install seaborn==0.11.0
class EnsembledModel() :
def __init__(self, model_paths):
super().__init__()
self.num_models = len(model_paths)
 self.leafmodel1 = get_leaf_model(model_paths[0])
 self.leafmodel2 = get_leaf_model(model_paths[1])
 self.effb4_model1 = get_efficient_b4_model(model_paths[2])
 self.effb4_model2 = get_efficient_b4_model(model_paths[3])
 self.effb4_model3 = get_efficient_b4_model(model_paths[4])
 self.effb4_model4 = get_efficient_b4_model(model_paths[5])
 self.effb4_model5 = get_efficient_b4_model(model_paths[6])
 self.effb7_model1 = get_efficient_b7_model(model_paths[7])
 self.effb7_model2 = get_efficient_b7_model(model_paths[8])
 self.effb7_model3 = get_efficient_b7_model(model_paths[9])
 self.res50_model1 = get_resnext_model(model_paths[10])
 self.res50_model2 = get_resnext_model(model_paths[11])
 self.res50_model3 = get_resnext_model(model_paths[12])
 self.res50_model4 = get_resnext_model(model_paths[13])
 self.res50_model5 = get_resnext_model(model_paths[14])
 self.res50_lr_model1 = get_resnext_model(model_paths[15])
 self.res50_lr_model2 = get_resnext_model(model_paths[16])
 self.res50_lr_model3 = get_resnext_model(model_paths[17])
 def predict(self, x):
with torch.no_grad() :
l1 = self.leafmodel1(x)
 l2 = self.leafmodel2(x)
 b4_e1 = self.effb4_model1(x)
 b4_e2 = self.effb4_model2(x)
 b4_e3 = self.effb4_model3(x)
 b4_e4 = self.effb4_model4(x)
 b4_e5 = self.effb4_model5(x)
 r1 = self.res50_model1(x)
 r2 = self.res50_model2(x)
 r3 = self.res50_model3(x)
 r4 = self.res50_model4(x)
 r5 = self.res50_model5(x)
 b7_e1 = self.effb7_model1(x)
 b7_e2 = self.effb7_model2(x)
 b7_e3 = self.effb7_model3(x)
 lr_r1 = self.res50_lr_model1(x)
 lr_r2 = self.res50_lr_model2(x)
 lr_r3 = self.res50_lr_model3(x)
 pred =(l1+l2 + b4_e1+b4_e2+b4_e3+b4_e4+b4_e5 + b7_e1+b7_e2+b7_e3 + r1+r2+r3+r4+r5 + lr_r1+lr_r2+lr_r3)/(self.num_models)
 return pred<define_variables>
warnings.filterwarnings("ignore")
 %matplotlib inline
red = Fore.RED
grn = Fore.GREEN
blu = Fore.BLUE
ylw = Fore.YELLOW
wht = Fore.WHITE
model_paths = [
'.. /input/cassava-trained-models/with_torch_crossentropy_LeafDiseasesModel Eff-4_fold-1.pt',
'.. /input/cassava-trained-models/with_torch_crossentropy_LeafDiseasesModel Eff-4_fold-5.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-4_fold-1.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-4_fold-2.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-4_fold-3.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-4_fold-4.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-4_fold-5.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-7_fold-1.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-7_fold-2.pt',
'.. /input/cassava-trained-models/LeafDiseasesModel Eff-7_fold-4.pt',
'.. /input/cassava-trained-models/Resnext50_fold-1.pth',
'.. /input/cassava-trained-models/Resnext50_fold-2.pth',
'.. /input/cassava-trained-models/Resnext50_fold-3.pth',
'.. /input/cassava-trained-models/Resnext50_fold-4.pth',
'.. /input/cassava-trained-models/Resnext50_fold-5.pth',
'.. /input/cassava-trained-models/Resnext50_with_lr_scheduler_fold-1.pth',
'.. /input/cassava-trained-models/Resnext50_with_lr_scheduler_fold-2.pth',
'.. /input/cassava-trained-models/Resnext50_with_lr_scheduler_fold-3.pth'
]
model = EnsembledModel(model_paths )<predict_on_test>
print(sns.__version__)
def inference(model, data_loader):
epoch_preds = 0
for epoch in range(CFG.epochs):
preds = []
for images in tqdm(data_loader):
images = images.to(device)
 logits = model.predict(images)
 preds += [logits.softmax(1 ).detach().cpu().numpy() ]
all_img_preds = np.concatenate(preds, axis=0)
 epoch_preds += all_img_preds
epoch_preds = epoch_preds / CFG.epochs
return epoch_preds<load_from_csv>
path = '.. /input/tabular-playground-series-jan-2021/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sample = pd.read_csv(path + 'sample_submission.csv' )
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test_data = TestDataset(test, transform=get_augmentation(data='test'))
test_loader = DataLoader(test_data, batch_size=CFG.batch_size, num_workers=CFG.num_workers )<prepare_output>
print('number of null columns in train set :- ',np.sum(train.isnull().sum() > 0))
print('number of null columns in test set :-',np.sum(test.isnull().sum() > 0))
predictions = inference(model, test_loader )<save_to_csv>
features = [f'cont{x}'for x in range(1,15)]
target = ['target']
test['label'] = predictions.argmax(1)
 test[['image_id', 'label']].to_csv(OUTPUT_DIR + 'submission.csv', index=False )<define_variables>
all_data = train[features+['id']].append(test )
BATCH_SIZE = 1
image_size = 512
enet_type = ['tf_efficientnet_b4_ns'] * 5
model_path = ['.. /input/moa-b4-baseline/baseline_cld_fold0_epoch8_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold1_epoch9_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold2_epoch9_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold3_epoch5_tf_efficientnet_b4_ns_512.pth',
'.. /input/moa-b4-baseline/baseline_cld_fold4_epoch11_tf_efficientnet_b4_ns_512.pth']<normalization>
lesser,greater = train[train['target'] <= train['target'].median() ],train[train['target'] >= train['target'].median() ]
transforms_valid = albumentations.Compose([
albumentations.CenterCrop(image_size, image_size, p=1),
albumentations.Resize(image_size, image_size),
albumentations.Normalize()
] )<define_variables>
train = train[train['target'] > 5]
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassava-resnext50-32x4d-weights/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'<define_search_space>
def rmse_score(yreal, yhat):
return sqrt(mean_squared_error(yreal, yhat))
class CFG:
debug=False
num_workers=8
model_name='resnext50_32x4d'
size=512
batch_size=32
seed=2020
target_size=5
target_col='label'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
inference=True<load_from_csv>
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
import optuna
from math import sqrt
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test['filepath'] = test.image_id.apply(lambda x: os.path.join('.. /input/cassava-leaf-disease-classification/test_images', f'{x}')) 
<create_dataframe>
train_ds = train.loc[:,features]
y_train = train.loc[:,target]
val_ds = test.loc[:,features]
test_dataset_efficient = CLDDataset(test, 'test', transform=transforms_valid)
 test_loader_efficient = torch.utils.data.DataLoader(test_dataset_efficient, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 )<categorify>
sc = MinMaxScaler()
train_ds = sc.fit_transform(train_ds)
 val_ds = sc.transform(val_ds )
def get_transforms(*, data):
if data == 'valid':
return A.Compose([
A.Resize(CFG.size, CFG.size),
A.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
] )<choose_model_class>
X, X_val, y, y_val = train_test_split(train_ds, y_train, test_size=0.35,random_state=43)
 print('Train', X.shape, y.shape, 'Test', X_val.shape, y_val.shape )
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x<define_search_model>
lgb_train = lgb.Dataset(X,y)
 lgb_valid = lgb.Dataset(X_val,y_val,reference=lgb_train )
class enet_v2(nn.Module):
def __init__(self, backbone, out_dim, pretrained=False):
super(enet_v2, self ).__init__()
self.enet = timm.create_model(backbone, pretrained=pretrained)
 in_ch = self.enet.classifier.in_features
self.myfc = nn.Linear(in_ch, out_dim)
 self.enet.classifier = nn.Identity()
def forward(self, x):
x = self.enet(x)
 x = self.myfc(x)
 return x<load_pretrained>
def objective(trial):
params = {
'num_leaves': trial.suggest_int('num_leaves', 32, 512),
'boosting_type': 'gbdt',
'max_bin': trial.suggest_int('max_bin', 700, 900),
'objective': 'regression',
'metric': 'RMSE',
'learning_rate': trial.suggest_float('learning_rate',0.0155,0.05),
'random_state' : 17,
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 16),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 8),
'min_child_samples': trial.suggest_int('min_child_samples', 4, 80),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 1.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 1.0),
'early_stopping_rounds': 10
}
model = lgb.train(params, lgb_train, valid_sets=[lgb_valid], verbose_eval=100, num_boost_round=800)
 y_pred = model.predict(X_val)
 score = rmse_score(y_val, y_pred)
 print(f"RMSE = {score}")
 return score
def load_state(model_path):
model = CustomResNext(CFG.model_name, pretrained=False)
 try:
model.load_state_dict(torch.load(model_path)['model'], strict=True)
 state_dict = torch.load(model_path)['model']
except:
state_dict = torch.load(model_path)['model']
state_dict = {k[7:] if k.startswith('module.')else k: state_dict[k] for k in state_dict.keys() }
return state_dict
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state)
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.softmax(1 ).to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<create_dataframe>
study = optuna.create_study(direction = 'minimize')
 study.optimize(objective,n_trials = 50 )
model = CustomResNext(CFG.model_name, pretrained=False)
 states = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 test_preds = []
for i in range(len(enet_type)) :
model = enet_v2(enet_type[i], out_dim=5)
 model = model.to(device)
 model.load_state_dict(torch.load(model_path[i]))
test_preds += [tta_inference_func(test_loader_efficient)]
pred = 0.8*predictions + 0.2*np.mean(test_preds, axis=0)
 test['label'] = softmax(pred ).argmax(1)
 test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)
 test.head()<define_variables>
print(f"Trials : {len(study.trials)}")
 print(f"Best Trial : {study.best_trial}")
 best = study.best_trial
for i, j in best.params.items() :
print(f"{i} : {j}")
 print(f"{best.value}" )
OUTPUT_DIR = "./"
MODEL_DIR = ".. /input/cassava-model/"
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = ".. /input/cassava-leaf-disease-classification/train_images"
TEST_PATH = ".. /input/cassava-leaf-disease-classification/test_images"<init_hyperparams>
best_params = best.params
;
class CFG:
debug = False
num_workers = 4
models = [
"tf_efficientnet_b4_ns",
"vit_base_patch16_384",
"seresnext50_32x4d",
]
size = {
"tf_efficientnet_b3_ns": 512,
"tf_efficientnet_b4_ns": 512,
"vit_base_patch16_384": 384,
"deit_base_patch16_384": 384,
"seresnext50_32x4d": 512,
}
batch_size = 64
seed = 7097
target_size = 5
target_col = "label"
n_fold = 5
trn_fold = {
"tf_efficientnet_b3_ns": {
"best": [0, 1, 2, 3, 4],
"final": [],
},
"tf_efficientnet_b4_ns": {
"best": [0, 1, 2, 3, 4],
"final": [],
},
"vit_base_patch16_384": {"best": [0, 1, 2, 3, 4], "final": []},
"deit_base_patch16_384": {"best": [0, 1, 2, 3, 4], "final": []},
"seresnext50_32x4d": {"best": [5, 6, 7, 8, 9], "final": []},
}
data_parallel = {
"tf_efficientnet_b3_ns": False,
"tf_efficientnet_b4_ns": True,
"vit_base_patch16_384": False,
"deit_base_patch16_384": False,
"seresnext50_32x4d": False,
}
transform = {
"tf_efficientnet_b4_ns": "rotate",
"vit_base_patch16_384": "rotate",
"seresnext50_32x4d": "rotate",
}
weight = {
"tf_efficientnet_b4_ns": 1,
"vit_base_patch16_384": 1,
"seresnext50_32x4d": 1,
}
tta = 10
no_tta_weight = tta - 1
train = False
inference = True<define_variables>
model = lgb.train(best_params, lgb_train, valid_sets=[lgb_train,lgb_valid],verbose_eval=50,num_boost_round=700 )
tta_weight_sum = CFG.no_tta_weight +(CFG.tta - 1)
 weight_sum = sum([CFG.weight[model] for model in CFG.models])* tta_weight_sum<load_from_csv>
preds = model.predict(X_val,num_iteration=model.best_iteration )
test = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 test.head()<choose_model_class>
print('LGBM: RMSE %.6f' %(rmse_score(y_val, preds)) )
class CassvaImgClassifier(nn.Module):
def __init__(self, model_name="resnext50_32x4d", pretrained=False):
super().__init__()
if model_name == "deit_base_patch16_384":
self.model = torch.hub.load(".. /input/fair-deit", model_name, pretrained=pretrained, source="local")
 n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, CFG.target_size)
 else:
self.model = timm.create_model(model_name, pretrained=pretrained)
 if "resnext50_32x4d" in model_name:
n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 elif model_name.startswith("tf_efficientnet"):
n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, CFG.target_size)
 elif model_name.startswith("vit_"):
n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x<choose_model_class>
train_preds = model.predict(train_ds )
def inference(model, states, test_loader, device, data_parallel):
model.to(device)
 if device == torch.device("cuda")and data_parallel:
model = torch.nn.DataParallel(model)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state["model"])
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.softmax(1 ).to("cpu" ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<choose_model_class>
print('LGBM: Train RMSE %.6f' %(rmse_score(y_train, train_preds)) )
predictions = None
for model_name in CFG.models:
for i in range(CFG.tta):
model = CassvaImgClassifier(model_name, pretrained=False)
 states = []
for saved_model in ["best", "final"]:
if CFG.trn_fold[model_name][saved_model] != []:
LOGGER.info(
 f"========== Model: {model_name}, TTA: {i}, Saved: {saved_model}, Fold: {CFG.trn_fold[model_name][saved_model]} ==========" 
)
 states += [
torch.load(MODEL_DIR + f"{model_name}_fold{fold}_{saved_model}.pth")
 for fold in CFG.trn_fold[model_name][saved_model]
]
if i == 0:
test_dataset = TestDataset(test, transform=get_transforms(data="valid", size=CFG.size[model_name]))
tta_weight = CFG.no_tta_weight
else:
test_dataset = TestDataset(
 test, transform=get_transforms(data=CFG.transform[model_name], size=CFG.size[model_name])
)
 tta_weight = 1
test_loader = DataLoader(
 test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True 
)
 inf = inference(model, states, test_loader, device, CFG.data_parallel[model_name])
 LOGGER.info(f"Inference example: {inf[0]}")
 if predictions is None:
predictions = inf[np.newaxis] * CFG.weight[model_name] * tta_weight
else:
predictions = np.append(predictions, inf[np.newaxis] * CFG.weight[model_name] * tta_weight, axis=0)
 sub = np.sum(predictions, axis=0)/ weight_sum
LOGGER.info(f"========== Overall ==========")
 LOGGER.info(f"Submission example: {sub[0]}")
 test["label"] = sub.argmax(1)
 test[["image_id", "label"]].to_csv(OUTPUT_DIR + "submission.csv", index=False)
 test.head()<install_modules>
train['t_preds'] = train_preds
!pip install -U /kaggle/input/kerasapplications<install_modules>
preds = model.predict(val_ds )
!pip install -U /kaggle/input/efficientnet/efficientnet-master<install_modules>
sub = pd.DataFrame({'id':test.id, 'target':preds})
 sub.to_csv('submission.csv',index=False )
<set_options><EOS> 
x = pd.read_csv('./submission.csv')
 x.head()