Kaggle competition: PetFinder.my Adoption Prediction
path = Path('.. /input/cassava-leaf-disease-classification' )<load_from_csv>
train_df = pd.read_csv(KAGGLE_DIR + "train/train.csv")
 test_df = pd.read_csv(KAGGLE_DIR + "test/test.csv")
 target = train_df['AdoptionSpeed']
train_df = pd.read_csv(path/'train.csv')
 train_df = train_df[~train_df['image_id'].isin(['1562043567.jpg', '3551135685.jpg', '2252529694.jpg'])]<prepare_x_and_y>
def metric(y1,y2):
return cohen_kappa_score(y1, y2, weights = 'quadratic')
 scorer = make_scorer(metric )
def get_x(r):
return path/'train_images'/r['image_id']
def get_y(r):
return r['label']<compute_train_metric>
target = train_df['AdoptionSpeed']
clean_df = train_df.drop(columns = ['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'])
 clean_test = test_df.drop(columns = ['Name', 'RescuerID', 'Description', 'PetID'] )
loss_func = LabelSmoothingCrossEntropy()<choose_model_class>
x_train, x_valid, y_train, y_valid = train_test_split(clean_df,
target,
test_size = 0.2,
random_state = seed)
 d_train = xgb.DMatrix(x_train, label = y_train)
 d_valid = xgb.DMatrix(x_valid, label = y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
learn = cnn_learner(dls, resnet50, metrics= [accuracy], loss_func=loss_func, opt_func=ranger, pretrained=False ).to_native_fp16()<load_pretrained>
gbm = GradientBoostingClassifier()
clf = RandomForestClassifier()
clf2 = ExtraTreesClassifier()
clf3 = AdaBoostClassifier()
xgb_params = {'objective' : 'multi:softmax',
'eval_metric' : 'mlogloss',
'eta' : 0.05,
'max_depth' : 4,
'num_class' : 5,
'lambda' : 0.8
}
gbm_grid = {
'loss' : ['deviance'],
'learning_rate' : [.025, 0.5],
'max_depth': [5, 8],
'max_features': ['auto'],
'min_samples_leaf': [100],
'min_samples_split': [100],
'n_estimators': [100],
'subsample' : [.8],
'random_state' : [seed]
}
rand_forest_grid = {
'bootstrap': [True, False],
'max_depth': [50, 85],
'max_features': ['auto'],
'min_samples_leaf': [10, 15],
'min_samples_split': [10, 15],
'n_estimators': [150, 200, 215],
'random_state' : [seed]
}
extra_trees_grid = {
'bootstrap' : [True, False],
'criterion' : ['gini'],
'max_depth' : [50, 75],
'max_features': ['auto'],
'min_samples_leaf': [10, 15],
'min_samples_split': [10, 15],
'n_estimators': [150, 200, 215],
'random_state' : [seed]
}
adaboost_grid = {
'n_estimators' : [150, 200, 225],
'learning_rate' : [.2],
'algorithm' : ['SAMME.R'],
'random_state' : [seed]
}
gbm_gridsearch = GridSearchCV(estimator = gbm,
param_grid = gbm_grid,
cv = 3,
n_jobs = -1,
verbose = 1,
scoring = scorer)
 rand_forest_gridsearch = GridSearchCV(estimator = clf,
param_grid = rand_forest_grid,
cv = 3,
n_jobs = -1,
verbose = 1,
scoring = scorer)
 extra_trees_gridsearch = GridSearchCV(estimator = clf2,
param_grid = extra_trees_grid,
cv = 3,
n_jobs = -1,
verbose = 1,
scoring = scorer)
 adaboost_gridsearch = GridSearchCV(estimator = clf3,
param_grid = adaboost_grid,
cv = 3,
n_jobs = -1,
verbose = 1,
scoring = scorer )
learn.load('/kaggle/input/leaf-resnet50/model_6' )<categorify>
print('Fitting XGBoost: ')
 bst = xgb.train(xgb_params,
d_train,
400,
watchlist,
early_stopping_rounds = 50,
verbose_eval = 0 )
learn = learn.to_native_fp32()<load_from_csv>
print('Fitting GBM: ')
 gbm_gridsearch.fit(clean_df, target)
 print('Fitting Random Forest: ')
 rand_forest_gridsearch.fit(clean_df, target)
 print('Fitting Extra Trees: ')
 extra_trees_gridsearch.fit(clean_df, target)
 print('Fitting AdaBoost: ')
 adaboost_gridsearch.fit(clean_df, target )
submission_df = pd.read_csv(path/'sample_submission.csv')
 submission_df.head()<feature_engineering>
print('Best model parameters:
')
 print('Gradient Boosting model:
{}
'.format(gbm_gridsearch.best_params_))
print('Random Forest model:
{}
'.format(rand_forest_gridsearch.best_params_))
print('Extra Trees model:
{}
'.format(extra_trees_gridsearch.best_params_))
print('Adaboost model:
{}
'.format(adaboost_gridsearch.best_params_))
test_data_path = submission_df['image_id'].apply(lambda x: path/'test_images'/x)
 tst_dl = learn.dls.test_dl(test_data_path)
 predictions = learn.tta(dl = tst_dl, n=10)
 submission_df['label'] = np.argmax(predictions[0],axis=1)
 submission_df<save_to_csv>
models = {'XGBoost' : bst,
'Gradient Boosting' : gbm_gridsearch,
'Random Forest' : rand_forest_gridsearch,
'Extra Trees' : extra_trees_gridsearch,
'Adaboost' : adaboost_gridsearch}
print('Score on the training set.This allows us to spot overfitting, performance, etc.( Rounded to 4 decimals):
')
 train_scores = []
for name, model in models.items() :
if name == 'XGBoost':
score = metric(bst.predict(xgb.DMatrix(clean_df)).astype(int), target)
 print('{} score: {}'.format(str(name), round(score, 4)))
 else:
score = metric(model.predict(clean_df), target)
 print('{} score: {}'.format(str(name), round(score, 4)))
 train_scores.append(score)
 print('
Mean Score: {0:10.4f}'.format(np.mean(train_scores)))
 print('
Standard Deviation of Scores: {0:10.4f}'.format(np.std(train_scores)) )
submission_df.to_csv('submission.csv',index=False )<train_model>
val_GBM = list(cross_val_score(gbm_gridsearch,
clean_df,
target,
scoring = scorer,
cv = 5))
val_RF = list(cross_val_score(rand_forest_gridsearch,
clean_df,
target,
scoring = scorer,
cv = 5))
val_ET = list(cross_val_score(extra_trees_gridsearch,
clean_df,
target,
scoring = scorer,
cv = 5))
val_ADA = list(cross_val_score(adaboost_gridsearch,
clean_df,
target,
scoring = scorer,
cv = 5))
val_XGB = metric(bst.predict(xgb.DMatrix(x_valid)).astype(int), y_valid )
kaggle = True
mode = 'training'
debug = True<import_modules>
print('Cross validation scores:
')
 print('Validation Score XGBoost:
{}
'.format(val_XGB))
print('Cross validation Gradient Boosting:
{},
Mean score: {}
Std of scores: {}
'.format([round(elem, 4)for elem in val_GBM],
round(np.mean(val_GBM), 4),
round(np.std(val_GBM), 4)))
 print('Cross validation Random Forest:
{},
Mean score: {}
Std of scores: {}
'.format([round(elem, 4)for elem in val_RF],
round(np.mean(val_RF), 4),
round(np.std(val_RF), 4)))
 print('Cross validation Extra Trees:
{},
Mean score: {}
Std of scores: {}
'.format([round(elem, 4)for elem in val_ET],
round(np.mean(val_ET), 4),
round(np.std(val_ET), 4)))
 print('Cross validation AdaBoost:
{},
Mean score: {}
Std of scores: {}
'.format([round(elem, 4)for elem in val_ADA],
round(np.mean(val_ADA), 4),
round(np.std(val_ADA), 4)))
 print('Mean Validation score: {}'.format(round(np.mean([np.mean(val_GBM),
np.mean(val_RF),
np.mean(val_ET),
np.mean(val_ADA),
val_XGB]), 4)))
 print('Standard Deviation of Cross Validation scores: {}'.format(round(np.std([np.mean(val_GBM),
np.mean(val_RF),
np.mean(val_ET),
np.mean(val_ADA),
val_XGB]), 4)))
import os
import pandas as pd
import numpy as np
import cv2
import random
import time
import gc
import json
import numbers
import copy
import matplotlib.pyplot as plt
import seaborn as sns
from functools import partial
from collections import OrderedDict
from pathlib import Path
from PIL import Image
from tqdm import tqdm
from pycm import ConfusionMatrix
import timm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from torch.cuda.amp import autocast, GradScaler
import torchvision.transforms as transforms
import albumentations as A
from albumentations.pytorch import ToTensorV2
from sklearn import metrics
from sklearn.model_selection import StratifiedKFold
from losses import LabelSmoothing, MyCrossEntropyLoss<define_variables>
pred0 = gbm_gridsearch.predict(clean_test)
 pred1 = rand_forest_gridsearch.predict(clean_test)
 pred2 = extra_trees_gridsearch.predict(clean_test)
 pred3 = adaboost_gridsearch.predict(clean_test)
 pred4 = bst.predict(xgb.DMatrix(clean_test)).astype(int)
 final_predictions = []
for pred in zip(pred0, pred1, pred2, pred3, pred4):
final_predictions.append(int(round(( sum(pred)) / len(pred), 0)) )
if kaggle:
data_dir = '/kaggle/input/cassava-leaf-disease-classification'
cache_dir = '/kaggle/working'
input_dir = '/kaggle/input'
else:
data_dir = '.. /.. /.. /data/cassava'
cache_dir = './'
input_dir = './cache'
train_images_dir = Path(data_dir)/ 'train_images'
test_images_dir = Path(data_dir)/ 'test_images'
save_weights_dir = Path(cache_dir)/ 'weights'
load_weights_dir = Path(input_dir)/ 'weights'
os.makedirs(str(save_weights_dir), exist_ok=True )<define_variables>
prediction_df = pd.DataFrame({'PetID' : test_df['PetID'],
'Gradient Boosting' : pred0,
'Random Forest' : pred1,
'Extra Trees' : pred2,
'Adaboost' : pred3,
'XGBoost' : pred4
})
 print('Predictions for each model: ')
 prediction_df.head(10 )
class GlobalConfig:
device_ids = [0, 1]
device = torch.device(f'cuda:{device_ids[0]}' if torch.cuda.is_available() else "cpu")
 image_model_name = 'tf_efficientnet_b4_ns'
load_weights = False
use_multi_gpus = False
num_workers = 8
batch_size = 16
num_folds = 2 if debug else 5
num_epochs = 3 if debug else 20
seed = 42
shuffle = False
onehot_cols = ['label_0', 'label_1', 'label_2', 'label_3', 'label_4']
num_targets = len(onehot_cols)
 use_tta = True
weights_mode = 'loss'
class DataConfig:
batch_size = GlobalConfig.batch_size
num_workers = GlobalConfig.num_workers
onehot_cols = GlobalConfig.onehot_cols
scale_size =(512, 512)
 input_size =(512, 512)
 trans_params = {
'interpolation': 'BILINEAR',
'random_resized_crop_scale':(0.08, 1.0),
'random_resized_crop_ratio':(0.75, 1.3333333333333333),
'rgb_mean':(0.485, 0.456, 0.406),
'rgb_std':(0.229, 0.224, 0.225),
'hue_shift_limit':(-10, 10),
'sat_shift_limit':(-15, 15),
'val_shift_limit':(-10, 10),
'brightness_limit':(-0.05, 0.05),
'contrast_limit':(-0.05, 0.05)
 }
class ModelConfig:
image_model_name = GlobalConfig.image_model_name
num_targets = GlobalConfig.num_targets
if kaggle:
json_path = f"/kaggle/input/timm-pretrained-efficientnet/index.json"
weights_dict = {}
with open(json_path, mode="r")as f:
weights_dict = json.load(f)
 weight_name = weights_dict['efficientnet'][f'{image_model_name}']
path = f"/kaggle/input/timm-pretrained-efficientnet/efficientnet/{weight_name}"
class FitterConfig:
device = GlobalConfig.device
num_epochs = GlobalConfig.num_epochs
batch_size = GlobalConfig.batch_size
onehot_cols = GlobalConfig.onehot_cols
use_tta = GlobalConfig.use_tta
weights_mode = GlobalConfig.weights_mode
num_targets = len(onehot_cols)
 num_augs_tta = 3
early_stopping_start = 0 if debug else 8
early_stopping_rounds = 4
iters_to_accumulate = 2
change_device_config = True
train_criterion = LabelSmoothing(smoothing=0.05)
 valid_criterion = MyCrossEntropyLoss()
optimizer = partial(torch.optim.Adam, lr=1e-4, weight_decay=1e-6)
 scheduler = partial(torch.optim.lr_scheduler.ReduceLROnPlateau, mode='max', patience=2, factor=0.2)
 step_scheduler = False
valid_scheduler = True
scheduler_update_by_loss = True<load_from_csv>
submission_df = pd.DataFrame(data = {'PetID' : test_df['PetID'],
'AdoptionSpeed' : final_predictions})
 submission_df.to_csv('submission.csv', index = False )