Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
print(os.listdir(".. /input"))
pd.set_option('display.max_columns', None)
 pd.options.display.float_format = '{:.4f}'.format
if not sys.warnoptions:
warnings.simplefilter("ignore")
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<split>
test = pd.read_csv('.. /input/test.csv')
 train = pd.read_csv('.. /input/train.csv')
 test.shape,train.shape
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
del model
torch.cuda.empty_cache()<feature_engineering>
train.isnull().sum()
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
test.isnull().sum()
test.to_csv('submission.csv', index=False )<install_modules>
print('Train','
',(train.isnull().sum() /len(train)) *100)
 print('-'*20)
 print('Test','
',(test.isnull().sum() /len(test)) *100 )
!pip install timm<set_options>
train.groupby(['Pclass'], as_index = False)['Age'].median()
set_seed(314 )<categorify>
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age
class CutMix(MixUp):
def __init__(self, alpha=1.) : self.distrib = Beta(tensor(alpha), tensor(alpha))
def before_batch(self):
lam = self.distrib.sample().squeeze().to(self.x.device)
 shuffle = torch.randperm(self.y.size(0)).to(self.x.device)
 self.yb1 = tuple(L(self.yb ).itemgot(shuffle))
nx_dims = len(self.x.size())
 bs, c, h, w = self.x.shape
rx, ry = w*self.distrib.sample() , h*self.distrib.sample()
rw, rh = w*(1-lam ).sqrt() , h*(1-lam ).sqrt()
x1 =(rx-rw/2 ).clamp(min=0 ).round().to(int)
 x2 =(rx+rw/2 ).clamp(max=w ).round().to(int)
 y1 =(ry-rh/2 ).clamp(min=0 ).round().to(int)
 y2 =(ry+rh/2 ).clamp(max=h ).round().to(int)
 self.learn.xb[0][:,:,y1:y2,x1:x2] = self.learn.xb[0][shuffle,:,y1:y2,x1:x2]
self.lam = 1- float(x2-x1)*(y2-y1)/(h*w)
 if not self.stack_y:
ny_dims = len(self.y.size())
 self.learn.yb = tuple(L(self.yb1,self.yb ).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=ny_dims-1)) )<feature_engineering>
train['Age'] = train[['Age','Pclass']].apply(lambda x: impute_age(x), axis = 1)
 test['Age'] = test[['Age','Pclass']].apply(lambda x: impute_age(x), axis = 1 )
df = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 df['image_id'] = df['image_id'].apply(lambda x: f'train_images/{x}')
 df.head()
idx2label = json.load(open('.. /input/cassava-leaf-disease-classification/label_num_to_disease_map.json'))
df.label = df.label.map(str ).map(idx2label)
 idxs = L.range(len(df)).shuffle()
df['valid'] = False
df.loc[idxs[:4279], 'valid'] = True<load_pretrained>
train['Embarked'].value_counts()
dls = ImageDataLoaders.from_df(df, path='.. /input/cassava-leaf-disease-classification/', bs=64,
item_tfms=Resize(320),
valid_col='valid',
batch_tfms=aug_transforms(size=300, mult=2, max_zoom=2.))
 dls.show_batch()<choose_model_class>
train['Embarked'] = np.where(~train['Embarked'].isnull() ,train['Embarked'],'S' )
model = timm.create_model('tf_efficientnet_b3_ns', pretrained=False )<load_from_disk>
test['Fare'] = np.where(~test['Fare'].isnull() ,test['Fare'],test['Fare'].median() )
model.load_state_dict(torch.load('.. /input/timm-pretrained-efficientnet/efficientnet/tf_efficientnet_b3_ns-9d44bf68.pth'))<choose_model_class>
print('Train','
',(train.isnull().sum() /len(train)) *100)
 print('-'*20)
 print('Test','
',(test.isnull().sum() /len(test)) *100 )
model.classifier = nn.Linear(model.classifier.in_features, len(dls.vocab))
learn = Learner(dls, model, loss_func=LabelSmoothingCrossEntropy() , splitter=methodcaller('parameters'), metrics=accuracy, model_dir='/kaggle/working/models')
 learn.to_fp16()
learn.freeze()<find_best_params>
train['Cabinknown'] = 0
train['Cabinknown'] = np.where(( train['Cabin'].isnull()),train['Cabinknown'],1)
 test['Cabinknown'] = 0
test['Cabinknown'] = np.where(( test['Cabin'].isnull()),test['Cabinknown'],1)
 train[['Cabinknown','Survived']].groupby('Cabinknown' ).mean()
learn.lr_find()<find_best_params>
train['Cabinletter'] = train['Cabin'].str[0]
test['Cabinletter'] = test['Cabin'].str[0]
learn.fine_tune(16, base_lr=8.3e-4, cbs=[ShowGraphCallback() , CutMix() ] )<feature_engineering>
train['Title'] = train['Name'].str.split(r'\s*,\s*|\s*\.\s*' ).str[1]
test['Title'] = test['Name'].str.split(r'\s*,\s*|\s*\.\s*' ).str[1]
dl = dls.valid
a1, target = learn.tta(dl=dl, n=16)
 pred_1 = a1.argmax(dim=1)
(pred_1==target ).to(float ).mean()<load_from_csv>
train['Family'] = train['SibSp'] + train['Parch'] + 1
test['Family'] = test['SibSp'] + test['Parch'] + 1
sample_df = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 sample_df.head()
sample_copy = sample_df.copy()
sample_copy['image_id'] = sample_copy['image_id'].apply(lambda x: f'test_images/{x}')
 test_dl = learn.dls.test_dl(sample_copy)
 test_dl.show_batch()<feature_engineering>
train['Agegroup'] = np.where(train['Age']<17,0,(np.where(( train['Age']>17)&(train['Age']<55),1,2)))
 test['Agegroup'] = np.where(test['Age']<17,0,(np.where(( test['Age']>17)&(test['Age']<55),1,2)) )
a, _ = learn.tta(dl=test_dl, n=16)
 pred = a.argmax(dim=1 ).numpy()
sample_df['label'] = pred<save_to_csv>
train['Faregroup'] = np.where(train['Fare']<15,0,1)
 test['Faregroup'] = np.where(test['Fare']<15,0,1 )
sample_df.to_csv('submission.csv',index=False )<import_modules>
train['Famgroup'] = np.where(train['Family']<2,0,(np.where(( train['Family']>1)&(train['Family']<5),1,2)))
 test['Famgroup'] = np.where(test['Age']<2,0,(np.where(( test['Family']>1)&(test['Family']<5),1,2)) )
<install_modules>
traina = train.drop(columns=['Name','Ticket','Cabin','Fare','Age','Family','SibSp','Parch'])
 testa = test.drop(columns=['Name','Ticket','Cabin','Fare','Age','Family','SibSp','Parch'] )
!pip install.. /input/ramki-cassava-weights/vision_transformer_pytorch-1.0.2-py2.py3-none-any.whl<define_variables>
TestId=testa['PassengerId']
total_features=pd.concat(( traina.drop(['PassengerId','Survived'], axis=1), testa.drop(['PassengerId'], axis=1)))
 total_features=pd.get_dummies(total_features, drop_first=True)
 train_features=total_features[0:traina.shape[0]]
test_features=total_features[traina.shape[0]:]
TRAINING = False
WEIGHT_FILE = '.. /input/ramki-cassava-weights/fold-4-weight-at-epoch-4-acc-0.95957.pth'<import_modules>
X = train_features
y = traina['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=101 )
import numpy as np
import pandas as pd
import os
from PIL import Image, ImageFilter
import cv2
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from torch.optim import *
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split, StratifiedKFold
from torchvision import models
import time
from tqdm.auto import tqdm
import random
from torch.optim.lr_scheduler import StepLR
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
import seaborn as sns
from torch.utils.tensorboard import SummaryWriter
from torch.utils.data.sampler import WeightedRandomSampler
from sklearn.metrics import classification_report
import torchvision.models as models
from vision_transformer_pytorch import VisionTransformer 
<import_modules>
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier()
gbc = GradientBoostingClassifier()
svc = SVC(probability=True)
 ext = ExtraTreesClassifier()
ada = AdaBoostClassifier()
gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier()
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores.append(acc.mean() )
from albumentations.pytorch import ToTensorV2
from albumentations import Rotate
import albumentations as A 
<set_options>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Score': scores})
 result_df = results.sort_values(by='Score', ascending=False ).reset_index(drop=True)
 result_df
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 device<set_options>
xgb_imp = pd.DataFrame({'Feature':train_features.columns, 'xgb importance':xgb.feature_importances_})
 gbc_imp = pd.DataFrame({'Feature':train_features.columns, 'gbc importance':gbc.feature_importances_})
 ran_imp = pd.DataFrame({'Feature':train_features.columns, 'ran importance':ran.feature_importances_})
 ext_imp = pd.DataFrame({'Feature':train_features.columns, 'ext importance':ext.feature_importances_})
 ada_imp = pd.DataFrame({'Feature':train_features.columns, 'ada importance':ada.feature_importances_})
 importances = gbc_imp.merge(xgb_imp, on='Feature' ).merge(ran_imp, on='Feature' ).merge(ext_imp, on='Feature' ).merge(ada_imp, on='Feature')
 importances['Average'] = importances.mean(axis=1)
 importances = importances.sort_values(by='Average', ascending=False ).reset_index(drop=True)
 importances
if torch.cuda.is_available() :
print(torch.cuda.device_count() )<define_variables>
columns = importances[importances['Average']<.03]['Feature'].values
train_features.drop(columns = columns, inplace = True)
 test_features.drop(columns = columns, inplace = True )
logs='logs/all-vti-tta-fold-'<init_hyperparams>
X = train_features
y = traina['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=101 )
SEED = 2286
N_FOLDS = 5
N_EPOCHS = 10
BATCH_SIZE = 16
IMG_SIZE = 384
LR = 1e-4
NUM_CLASSES = 5
OPTM_STEP = 1
TTA=5
EVAL_TTA_EVERY=5 
<set_options>
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier(random_state=1)
 gbc = GradientBoostingClassifier(random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(random_state=1)
 ada = AdaBoostClassifier(random_state=1)
 gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v2 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v2.append(acc.mean() )
def seed_everything(seed):

 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<define_variables>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Score': scores,
'Score w/Feature Selection': scores_v2})
 result_df = results.sort_values(by='Score w/Feature Selection', ascending=False ).reset_index(drop=True)
 result_df
base_path = '.. /input/cassava-leaf-disease-classification/' 
<load_from_csv>
penalty = ['l1', 'l2']
C = np.logspace(0, 4, 10)
 hyperparams = {'penalty': penalty, 'C': C}
lrgd=GridSearchCV(estimator = LogisticRegression() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 lrgd.fit(X_train, y_train)
 print(lrgd.best_score_)
 print(lrgd.best_estimator_ )
train_path =base_path + 'train_images/'
test_path = base_path + 'test_images/'
train_csv = pd.read_csv(base_path + 'train.csv')
 sample = pd.read_csv(base_path + 'sample_submission.csv' )<count_values>
n_neighbors = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20]
algorithm = ['auto']
weights = ['uniform', 'distance']
leaf_size = [1, 2, 3, 4, 5, 10, 15, 20, 25, 30]
hyperparams = {'algorithm': algorithm, 'weights': weights, 'leaf_size': leaf_size,
'n_neighbors': n_neighbors}
kngd=GridSearchCV(estimator = KNeighborsClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 kngd.fit(X_train, y_train)
 print(kngd.best_score_)
 print(kngd.best_estimator_ )
train_csv_disease = train_csv.label.map({0:"Cassava Bacterial Blight(CBB)",
1:"Cassava Brown Streak Disease(CBSD)",
2:"Cassava Green Mottle(CGM)",
3:"Cassava Mosaic Disease(CMD)",
4:"Healthy"})
 diseases = train_csv_disease.value_counts()<create_dataframe>
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
rfgd=GridSearchCV(estimator = RandomForestClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 rfgd.fit(X_train, y_train)
 print(rfgd.best_score_)
 print(rfgd.best_estimator_ )
class CasavaDataset(Dataset):
def __init__(self, dataframe, transforms=None, test=False):
self.df = dataframe
self.transforms = transforms
self.test = test
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
label = self.df.iloc[idx].label
p = self.df.iloc[idx].image_id
if self.test == False:
p_path = train_path + p
else:
p_path = test_path + p
image = Image.open(p_path ).convert('RGB')
 image = np.array(image)
 if self.transforms:
transformed = self.transforms(image=image)
 image = transformed['image']
return image, label<choose_model_class>
Cs = [0.001, 0.01, 0.1, 1, 5, 10, 15, 20, 50, 100]
gammas = [0.001, 0.01, 0.1, 1]
hyperparams = {'C': Cs, 'gamma' : gammas}
svgd=GridSearchCV(estimator = SVC(probability=True), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 svgd.fit(X_train, y_train)
 print(svgd.best_score_)
 print(svgd.best_estimator_ )
folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)
<choose_model_class>
n_restarts_optimizer = [0, 1, 2, 3]
max_iter_predict = [1, 2, 5, 10, 20, 35, 50, 100]
warm_start = [True, False]
hyperparams = {'n_restarts_optimizer': n_restarts_optimizer, 'max_iter_predict': max_iter_predict, 'warm_start': warm_start}
gpgd=GridSearchCV(estimator = GaussianProcessClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 gpgd.fit(X_train, y_train)
 print(gpgd.best_score_)
 print(gpgd.best_estimator_ )
class CassvaModel(nn.Module):
def __init__(self, model_arch='tf_efficientnet_b7_ns', n_class=5, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 in_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(in_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<choose_model_class>
n_estimators = [10, 25, 50, 75, 100, 125, 150, 200]
learning_rate = [0.001, 0.01, 0.1, 0.5, 1, 1.5, 2]
hyperparams = {'n_estimators': n_estimators, 'learning_rate': learning_rate}
adgd=GridSearchCV(estimator = AdaBoostClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 adgd.fit(X_train, y_train)
 print(adgd.best_score_)
 print(adgd.best_estimator_ )
def build_model() :
if TRAINING:
model = VisionTransformer.from_pretrained('ViT-B_16', num_classes=5)
 print('model created with imagenet weights')
 else:
model = VisionTransformer.from_name('ViT-B_16', num_classes=5)
 print('model loaded from random weight')
 return model.to(device )<create_dataframe>
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [100, 250, 500, 750, 1000, 1250, 1500]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gbgd=GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 gbgd.fit(X_train, y_train)
 print(gbgd.best_score_)
 print(gbgd.best_estimator_ )
testset = CasavaDataset(sample, transforms=valid_transforms, test=True)
 test_loader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=1 )<feature_engineering>
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
etgd=GridSearchCV(estimator = ExtraTreesClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 etgd.fit(X_train, y_train)
 print(etgd.best_score_)
 print(etgd.best_estimator_ )
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<train_on_grid>
n_estimators = [10, 15, 20, 25, 50, 75, 100, 150]
max_samples = [1, 2, 3, 5, 7, 10, 15, 20, 25, 30, 50]
max_features = [1, 3, 5, 7]
hyperparams = {'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features}
bcgd=GridSearchCV(estimator = BaggingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs =-1)
 bcgd.fit(X_train, y_train)
 print(bcgd.best_score_)
 print(bcgd.best_estimator_ )
def train_model(model, epoch, dataloader_train, criterion, optimizer):
model.train()
losses = AverageMeter()
accs = AverageMeter()
tk = tqdm(dataloader_train, total=len(dataloader_train), position=0, leave=True)
 optimizer.zero_grad()
for idx,(img, labels)in enumerate(tk):
images, labels = img.to(device), labels.to(device ).long()
predicted = model(images)
 loss = criterion(predicted, labels)
 loss.backward()
if(idx+1)% OPTM_STEP == 0:
optimizer.step()
optimizer.zero_grad()
predicted_classes = predicted.argmax(1)
 correctly_identified_sum =(predicted_classes==labels ).sum().item()
number_of_images = images.size(0)
 accs.update(correctly_identified_sum/number_of_images, number_of_images)
 losses.update(loss.item() , number_of_images)
 tk.set_postfix(loss=losses.avg,acc=accs.avg)
 return losses.avg, accs.avg
def test_model(model, dataloader_valid, criterion):
model.eval()
losses = AverageMeter()
accs = AverageMeter()
with torch.no_grad() :
tk = tqdm(dataloader_valid, total=len(dataloader_valid), position=0, leave=True)
 for idx,(images, labels)in enumerate(tk):
images, labels = images.to(device), labels.to(device ).long()
output_valid = model(images)
 loss = criterion(output_valid, labels)
 losses.update(loss.item() , images.size(0))
accs.update(( output_valid.argmax(1)==labels ).sum().item() /images.size(0),images.size(0))
tk.set_postfix(loss=losses.avg,acc=accs.avg)
 return losses.avg,accs.avg , loss<choose_model_class>
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [10, 25, 50, 75, 100, 250, 500, 750, 1000]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = XGBClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
def tta_model(model, dataloader_tta):
model.eval()
losses = AverageMeter()
accs = AverageMeter()
pred = []
all_labels = []
with torch.no_grad() :
for tta_idx in tqdm(range(TTA)) :
avg_preds = []
for idx,(images, labels)in enumerate(dataloader_tta):
if tta_idx==0:
all_labels.extend(labels.tolist())
 images, labels = images.to(device), labels.to(device ).long()
output = model(images)
 avg_preds.append(output.softmax(1 ).to('cpu' ).detach().numpy())
 all_pred = np.concatenate(avg_preds)
 pred.append(all_pred)
 pred = np.mean(pred, axis=0)
 acc =(pred.argmax(1)==all_labels ).sum().item() /pred.shape[0]
return acc, pred<prepare_x_and_y>
max_depth = [3, 4, 5, 6, 7, 8, 9, 10]
min_child_weight = [1, 2, 3, 4, 5, 6]
hyperparams = {'max_depth': max_depth, 'min_child_weight': min_child_weight}
gd=GridSearchCV(estimator = gd.best_estimator_, param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy",n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
X = train_csv.iloc[:,:-1]
y = train_csv.iloc[:,-1:]<count_unique_values>
gamma = [i*0.1 for i in range(0,5)]
hyperparams = {'gamma': gamma}
gd=GridSearchCV(estimator = gd.best_estimator_,param_grid = hyperparams, verbose=True, cv=5, scoring = "accuracy",n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
if TRAINING:
for i_fold,(train_idx, valid_idx)in enumerate(folds.split(X,y)) :
print("Fold {}/{}".format(i_fold + 1, N_FOLDS))
train = train_csv.iloc[train_idx]
train.reset_index(drop=True, inplace=True)
 valid = train_csv.iloc[valid_idx]
valid.reset_index(drop=True, inplace=True)
 unique_labels, nSamples = np.unique(train.iloc[:,-1:], return_counts =True)
 classWeights = [1 -(x / sum(nSamples)) for x in nSamples]
classWeights = torch.FloatTensor(classWeights ).to(device)
 dataset_train = CasavaDataset(train, transforms=train_transforms)
 dataset_valid = CasavaDataset(valid, transforms=valid_transforms)
 dataset_tta = CasavaDataset(valid, transforms=tta_inference_transforms)
 dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=1, shuffle=True)
 dataloader_valid = DataLoader(dataset_valid, batch_size=BATCH_SIZE, num_workers=1, shuffle=False)
 dataloader_tta = DataLoader(dataset_tta, batch_size=BATCH_SIZE, num_workers=1, shuffle=False)
 optimizer = torch.optim.AdamW(model.parameters() , lr=LR)
 criterion = nn.CrossEntropyLoss(weight=classWeights)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, \
patience=5, verbose=True, min_lr=1e-5)
 best_acc = 0
best_loss = 100
for epoch in range(1,N_EPOCHS+1):
train_loss, train_acc = train_model(model, epoch, dataloader_train, criterion, optimizer)
 val_loss, val_acc, loss = test_model(model, dataloader_valid, criterion)
 writer.add_scalar('training acc', train_acc, i_fold*epoch+epoch)
 writer.add_scalar('training loss', train_loss, i_fold*epoch+epoch)
 writer.add_scalar('validation loss', val_loss, i_fold*epoch+epoch)
 writer.add_scalar('validation Acc', val_acc, i_fold*epoch+epoch)
 writer.flush()
scheduler.step(val_acc)
 if epoch % EVAL_TTA_EVERY ==0:
tta_acc = tta_model(model, dataloader_tta)
 print("Acc -----> TTA:{} ==> Normal:{}", tta_acc,val_acc)
 if val_loss < best_loss:
best_loss = val_loss
torch.save(model.state_dict() , 'vti_weigthts/fold-{}-loss-weight-at-epoch-{}-loss-{:.5}.pth'.format(i_fold, epoch, best_loss))
if val_acc > best_acc:
best_acc = val_acc
torch.save(model.state_dict() , 'vti_weigthts/fold-{}-weight-at-epoch-{}-acc-{:.5}.pth'.format(i_fold, epoch, best_acc))
print('current_val_acc:', val_acc, 'best_val_acc:', best_acc)
 writer.close()
show_metrics(model, dataloader_train, criterion)
<load_pretrained>
subsample = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
colsample_bytree = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
hyperparams = {'subsample': subsample, 'colsample_bytree': colsample_bytree}
gd=GridSearchCV(estimator = gd.best_estimator_, param_grid = hyperparams, verbose=True, cv=5, scoring = "accuracy",n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
model = build_model()
model.load_state_dict(torch.load(WEIGHT_FILE, map_location='cuda:0'))
model = model.to(device )<save_to_csv>
reg_alpha = [1e-5, 1e-2, 0.1, 1, 100]
hyperparams = {'reg_alpha': reg_alpha}
xggd=GridSearchCV(estimator = gd.best_estimator_,param_grid = hyperparams, verbose=True, cv=5, scoring = "accuracy",n_jobs=-1)
 xggd.fit(X_train, y_train)
 print(xggd.best_score_)
 print(xggd.best_estimator_ )
test_pred = []
model.eval()
tta_acc, pred = tta_model(model, test_loader)
 pred = pred.argmax(1 ).astype('int')
 test_pred.extend(pred)
 sample.label = test_pred
sample.to_csv('submission.csv',index=False )<load_from_csv>
ran = rfgd.best_estimator_
knn = kngd.best_estimator_
log = lrgd.best_estimator_
xgb = xggd.best_estimator_
gbc = gbgd.best_estimator_
svc = svgd.best_estimator_
ext = etgd.best_estimator_
ada = adgd.best_estimator_
gpc = gpgd.best_estimator_
bag = bcgd.best_estimator_
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v3 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v3.append(acc.mean() )
!cat submission.csv<set_options>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2,
'Score with tuned parameters': scores_v3})
 result_df = results.sort_values(by='Score with tuned parameters', ascending=False ).reset_index(drop=True)
 result_df
InteractiveShell.ast_node_interactivity = "all"
train = False<import_modules>
ran = rfgd.best_estimator_
knn = kngd.best_estimator_
log = lrgd.best_estimator_
xgb = xggd.best_estimator_
gbc = gbgd.best_estimator_
svc = svgd.best_estimator_
ext = etgd.best_estimator_
ada = adgd.best_estimator_
gpc = gpgd.best_estimator_
bag = bcgd.best_estimator_
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v4 = []
for mod in models:
mod.fit(X_train, y_train)
 predict = mod.predict(X_test)
 acc = accuracy_score(y_test, predict)
 scores_v4.append(acc )
fastai.__version__<load_from_csv>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2,
'Score with tuned parameters': scores_v3,
'Score with full accuracy': scores_v4})
 result_df = results.sort_values(by='Score with full accuracy', ascending=False ).reset_index(drop=True)
 result_df
labels = pd.read_csv(path / 'train.csv')
 labels["image_id"] = labels["image_id"].apply(
 lambda x: f'train_images/{x}')
 labels.head()<load_pretrained>
predictions = ext.predict(test_features )
dls = ImageDataLoaders.from_df(
 labels,
path='.. /input/cassava-leaf-disease-classification/',
bs=64,
fn_col=0,
label_col=1,
seed=42,
valid_pct=0.2,
item_tfms=RandomResizedCrop(512, min_scale=0.75, ratio=(1.,1.)),
batch_tfms=aug_transforms() ,
num_workers=0)
 dls.valid_ds.items[:3]<choose_model_class>
submission=pd.DataFrame()
submission['PassengerId']=TestId
submission['Survived']=predictions
submission.to_csv('submission.csv', index=False )