Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<drop_column>
!pip install.. /input/timmwhl/timm-0.3.3-py3-none-any.whl
<categorify>
import random
import os
import sys
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torchvision
import timm
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
from tqdm import tqdm
import torch.nn.functional as F
import albumentations as A
from albumentations import Compose
from albumentations.pytorch import ToTensorV2
import cv2
for col in ['original_language', 'collection_name', 'all_genres']:
le = LabelEncoder()
le.fit(list(train[col].fillna('')) + list(test[col].fillna('')))
 train[col] = le.transform(train[col].fillna('' ).astype(str))
test[col] = le.transform(test[col].fillna('' ).astype(str))<define_variables>
warnings.filterwarnings("ignore" )
train_texts = train[['title', 'tagline', 'overview', 'original_title']]
test_texts = test[['title', 'tagline', 'overview', 'original_title']]<feature_engineering>
DATA_PATH = '.. /input/cassava-leaf-disease-classification/'
bs = 16
sz = 448
TIMM_MODEL = 'resnet50'
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train = train.drop(col, axis=1)
 test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test = test.drop(col, axis=1 )<feature_engineering>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
SEED = 1234
seed_everything(SEED)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'revenue'] = 6800000
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3197,'budget'] = 8000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
power_six = train.id[train.budget > 1000][train.revenue < 100]
for k in power_six :
train.loc[train['id'] == k,'revenue'] = train.loc[train['id'] == k,'revenue'] * 1000000<prepare_x_and_y>
class CassavaDataset(Dataset):
def __init__(self, dataframe, root_dir, transforms=None):
super().__init__()
self.dataframe = dataframe
self.root_dir = root_dir
self.transforms = transforms
def __len__(self):
return len(self.dataframe)
 def get_img_bgr_to_rgb(self, path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
img_name = os.path.join(self.root_dir,
self.dataframe.iloc[idx, 0])
 image = self.get_img_bgr_to_rgb(img_name)
 if self.transforms:
image = self.transforms(image=image)['image']
csv_row = self.dataframe.iloc[idx, 1:]
sample = {
'image': image,
'label': csv_row.label,
}
return sample
X = train.drop(['id', 'revenue','production_companies'], axis=1)
 y = np.log1p(train['revenue'])
 X_test = test.drop(['id','production_companies'], axis=1 )<split>
def test_transforms() :
return Compose([
A.Resize(sz, sz),
A.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1)
<create_dataframe>
class CassavaNet(nn.Module):
def __init__(self):
super().__init__()
backbone = timm.create_model(TIMM_MODEL, pretrained=False)
 n_features = backbone.fc.in_features
self.backbone = nn.Sequential(*backbone.children())[:-2]
self.classifier = nn.Linear(n_features, 5)
 self.pool = nn.AdaptiveAvgPool2d(( 1, 1))
def forward_features(self, x):
x = self.backbone(x)
 return x
def forward(self, x):
feats = self.forward_features(x)
 x = self.pool(feats ).view(x.size(0), -1)
 x = self.classifier(x)
 return x, feats
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
print('Start training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=10000,
valid_sets=lgb_eval,
early_stopping_rounds=100)
 print('Start predicting...')
 preds = gbm.predict(X_valid, num_iteration=gbm.best_iteration)
 for pred in preds:
result = prediction = int(np.argmax(pred))
'
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )<load_pretrained>
model = CassavaNet().to(device )
eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )<choose_model_class>
def predict(model, ckpts, dataloader):
predict_list=[]
with torch.no_grad() :
for _, data in enumerate(dataloader):
avg_preds = []
for ckpt in ckpts:
model.load_state_dict(ckpt['state_dict'])
 model.eval()
images, label = data.values()
images = images.to(device)
 outputs, _ = model(images)
 preds = F.softmax(outputs ).to('cpu' ).numpy()
avg_preds.append(preds)
 predict_list.append(np.mean(avg_preds, axis=0))
predict_list = np.concatenate(predict_list)
 return predict_list.argmax(axis=1 )
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 )<split>
test_df = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test_dir = '.. /input/cassava-leaf-disease-classification/test_images/'
test_ds = CassavaDataset(dataframe=test_df,
root_dir=test_dir,
transforms=test_transforms())
 test_dl = DataLoader(test_ds, batch_size=bs,
shuffle=False, num_workers=8,
pin_memory=True )
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(X.shape[0])
 prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction<train_model>
ckpts=[]
trained_model_path = ".. /input/cassavalblsmoothingresnet50"
for path in os.listdir(trained_model_path):
ckpts.append(torch.load(os.path.join(trained_model_path, path)))
params = {'num_leaves': 30,
'min_data_in_leaf': 10,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )<feature_engineering>
test_predict_list=predict(model, ckpts, test_dl )
<feature_engineering><EOS> 
test_df['label'] = test_predict_list
test_df[['image_id', 'label']].to_csv('submission.csv', index=False)
 test_df.head()