Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<load_from_csv>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv' )<define_variables>
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )
categorical_cols=['cat'+str(i)for i in range(10)]
continous_cols=['cont'+str(i)for i in range(14)]<categorify>
debug=False
submission=True
batch_size=256
device='cuda:0'
out='.'
for e in categorical_cols:
le = LabelEncoder()
train[e]=le.fit_transform(train[e])
 test[e]=le.transform(test[e] )<prepare_x_and_y>
datadir = Path('/kaggle/input/bengaliai-cv19')
 featherdir = Path('/kaggle/input/bengaliaicv19feather')
 outdir = Path('.' )
data=train[categorical_cols+continous_cols]
target=train['target']<split>
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)
 param = {
'metric': 'rmse',
'random_state': 48,
'n_estimators': 20000,
'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),
'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.006,0.008,0.01,0.014,0.017,0.02]),
'max_depth': trial.suggest_categorical('max_depth', [10,20,100]),
'num_leaves' : trial.suggest_int('num_leaves', 1, 1000),
'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),
'cat_smooth' : trial.suggest_int('min_data_per_groups', 1, 100)
 }
model = LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>

 class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<create_dataframe>
class BengaliAIDataset(DatasetMixin):
def __init__(self, images, labels=None, transform=None, indices=None):
super(BengaliAIDataset, self ).__init__(transform=transform)
 self.images = images
self.labels = labels
if indices is None:
indices = np.arange(len(images))
self.indices = indices
self.train = labels is not None
def __len__(self):

 return len(self.indices)
 def get_example(self, i):

 i = self.indices[i]
x = self.images[i]
x =(255 - x ).astype(np.float32)/ 255.
 if self.train:
y = self.labels[i]
return x, y
else:
return x
study.trials_dataframe()<find_best_params>
def add_gaussian_noise(x, sigma):
x += np.random.randn(*x.shape)* sigma
x = np.clip(x, 0., 1.)
return x
class Transform:
def __init__(self, affine=True, crop=True, size=(64, 64),
normalize=True, train=True, threshold=40.,
sigma=-1.) :
self.affine = affine
self.crop = crop
self.size = size
self.normalize = normalize
self.train = train
self.threshold = threshold / 255.
 self.sigma = sigma / 255.
 def __call__(self, example):
if self.train:
x, y = example
else:
x = example
if self.affine:
x = affine_image(x)
 if self.crop:
x = crop_char_image(x, threshold=self.threshold)
 if self.size is not None:
x = resize(x, size=self.size)
 if self.sigma > 0.:
x = add_gaussian_noise(x, sigma=self.sigma)
 if self.normalize:
x =(x.astype(np.float32)- 0.0692)/ 0.2051
if x.ndim == 2:
x = x[None, :, :]
x = x.astype(np.float32)
 if self.train:
y = y.astype(np.int64)
 return x, y
else:
return x
params=study.best_params
params['random_state'] = 48
params['n_estimators'] = 20000
params['metric'] = 'rmse'<drop_column>
def residual_add(lhs, rhs):
lhs_ch, rhs_ch = lhs.shape[1], rhs.shape[1]
if lhs_ch < rhs_ch:
out = lhs + rhs[:, :lhs_ch]
elif lhs_ch > rhs_ch:
out = torch.cat([lhs[:, :rhs_ch] + rhs, lhs[:, rhs_ch:]], dim=1)
 else:
out = lhs + rhs
return out
params['cat_smooth'] = params.pop('min_data_per_groups' )<find_best_model_class>
class LazyLoadModule(nn.Module):

 _lazy_buffer_keys: List[str] = []
_lazy_parameter_keys: List[str] = []
def __init__(self):
super(LazyLoadModule, self ).__init__()
for k in self._lazy_buffer_keys:
self.register_buffer(k, torch.tensor([]))
for k in self._lazy_parameter_keys:
self.register_parameter(k, None)
 self._register_load_state_dict_pre_hook(self._hook)
 def _hook(self, state_dict, prefix, local_metadata, strict, missing_keys,
unexpected_keys, error_msgs):
for key in self._lazy_buffer_keys:
self.register_buffer(key, state_dict[prefix + key])
 for key in self._lazy_parameter_keys:
self.register_parameter(key, Parameter(state_dict[prefix + key]))
columns = categorical_cols+continous_cols
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = LGBMRegressor(**params)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1<save_to_csv>
class LazyLinear(LazyLoadModule):

 __constants__ = ['bias', 'in_features', 'out_features']
_lazy_parameter_keys = ['weight']
def __init__(self, in_features, out_features, bias=True):
super(LazyLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
if bias:
self.bias = Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 if in_features is not None:
self.weight = Parameter(torch.Tensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight, a=math.sqrt(5))
if self.bias is not None:
fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
 bound = 1 / math.sqrt(fan_in)
 init.uniform_(self.bias, -bound, bound)
 def forward(self, input):
if self.weight is None:
self.in_features = input.shape[-1]
self.weight = Parameter(torch.Tensor(self.out_features, self.in_features))
self.reset_parameters()
self.to(input.device)
 return F.linear(input, self.weight, self.bias)
 def extra_repr(self):
return 'in_features={}, out_features={}, bias={}'.format(
 self.in_features, self.out_features, self.bias is not None 
)
sub['target']=preds
sub.to_csv('submission.csv', index=False )<import_modules>
class LinearBlock(nn.Module):
def __init__(self, in_features, out_features, bias=True,
use_bn=True, activation=F.relu, dropout_ratio=-1, residual=False,):
super(LinearBlock, self ).__init__()
if in_features is None:
self.linear = LazyLinear(in_features, out_features, bias=bias)
 else:
self.linear = nn.Linear(in_features, out_features, bias=bias)
 if use_bn:
self.bn = nn.BatchNorm1d(out_features)
 if dropout_ratio > 0.:
self.dropout = nn.Dropout(p=dropout_ratio)
 else:
self.dropout = None
self.activation = activation
self.use_bn = use_bn
self.dropout_ratio = dropout_ratio
self.residual = residual
def __call__(self, x):
h = self.linear(x)
 if self.use_bn:
h = self.bn(h)
 if self.activation is not None:
h = self.activation(h)
 if self.residual:
h = residual_add(h, x)
 if self.dropout_ratio > 0:
h = self.dropout(h)
 return h
import pandas as pd
import numpy as np
from lightgbm import LGBMRegressor
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error<load_from_csv>
class PretrainedCNN(nn.Module):
def __init__(self, model_name='se_resnext101_32x4d',
in_channels=1, out_dim=10, use_bn=True,
pretrained=None):
super(PretrainedCNN, self ).__init__()
self.conv0 = nn.Conv2d(
 in_channels, 3, kernel_size=3, stride=1, padding=1, bias=True)
 self.base_model = pretrainedmodels.__dict__[model_name](pretrained=pretrained)
 activation = F.leaky_relu
self.do_pooling = True
if self.do_pooling:
inch = self.base_model.last_linear.in_features
else:
inch = None
hdim = 512
lin1 = LinearBlock(inch, hdim, use_bn=use_bn, activation=activation, residual=False)
 lin2 = LinearBlock(hdim, out_dim, use_bn=use_bn, activation=None, residual=False)
 self.lin_layers = Sequential(lin1, lin2)
 def forward(self, x):
h = self.conv0(x)
 h = self.base_model.features(h)
 if self.do_pooling:
h = torch.sum(h, dim=(-1, -2))
else:
bs, ch, height, width = h.shape
h = h.view(bs, ch*height*width)
 for layer in self.lin_layers:
h = layer(h)
 return h
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv', index_col='id')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv', index_col='id' )<data_type_conversions>
def accuracy(y, t):
pred_label = torch.argmax(y, dim=1)
 count = pred_label.shape[0]
correct =(pred_label == t ).sum().type(torch.float32)
 acc = correct / count
return acc
class BengaliClassifier(nn.Module):
def __init__(self, predictor, n_grapheme=168, n_vowel=11, n_consonant=7):
super(BengaliClassifier, self ).__init__()
self.n_grapheme = n_grapheme
self.n_vowel = n_vowel
self.n_consonant = n_consonant
self.n_total_class = self.n_grapheme + self.n_vowel + self.n_consonant
self.predictor = predictor
self.metrics_keys = [
'loss', 'loss_grapheme', 'loss_vowel', 'loss_consonant',
'acc_grapheme', 'acc_vowel', 'acc_consonant']
def forward(self, x, y=None):
pred = self.predictor(x)
 if isinstance(pred, tuple):
assert len(pred)== 3
preds = pred
else:
assert pred.shape[1] == self.n_total_class
preds = torch.split(pred, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 loss_grapheme = F.cross_entropy(preds[0], y[:, 0])
 loss_vowel = F.cross_entropy(preds[1], y[:, 1])
 loss_consonant = F.cross_entropy(preds[2], y[:, 2])
 loss = loss_grapheme + loss_vowel + loss_consonant
metrics = {
'loss': loss.item() ,
'loss_grapheme': loss_grapheme.item() ,
'loss_vowel': loss_vowel.item() ,
'loss_consonant': loss_consonant.item() ,
'acc_grapheme': accuracy(preds[0], y[:, 0]),
'acc_vowel': accuracy(preds[1], y[:, 1]),
'acc_consonant': accuracy(preds[2], y[:, 2]),
}
return loss, metrics, pred
def calc(self, data_loader):
device: torch.device = next(self.parameters() ).device
self.eval()
output_list = []
with torch.no_grad() :
for batch in tqdm(data_loader):
batch = batch.to(device)
 pred = self.predictor(batch)
 output_list.append(pred)
 output = torch.cat(output_list, dim=0)
 preds = torch.split(output, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 return preds
def predict_proba(self, data_loader):
preds = self.calc(data_loader)
 return [F.softmax(p, dim=1)for p in preds]
def predict(self, data_loader):
preds = self.calc(data_loader)
 pred_labels = [torch.argmax(p, dim=1)for p in preds]
return pred_labels
cat_cols = [col for col in train.columns if 'cat' in col]
train[cat_cols] = train[cat_cols].astype('category')
 test[cat_cols] = test[cat_cols].astype('category' )<train_model>
def build_predictor(arch, out_dim, model_name=None):
if arch == 'pretrained':
predictor = PretrainedCNN(in_channels=1, out_dim=out_dim, model_name=model_name)
 else:
raise ValueError("[ERROR] Unexpected value arch={}".format(arch))
return predictor
def build_classifier(arch, load_model_path, n_total, model_name='', device='cuda:0'):
if isinstance(device, str):
device = torch.device(device)
 predictor = build_predictor(arch, out_dim=n_total, model_name=model_name)
 print('predictor', type(predictor))
classifier = BengaliClassifier(predictor)
 if load_model_path:
predictor.load_state_dict(torch.load(load_model_path))
else:
print("[WARNING] Unexpected value load_model_path={}" 
.format(load_model_path))
classifier.to(device)
 return classifier
class BaggingLGBM() :

 def __init__(self, n_estimators=5, max_sample=0.8, params=None):
params = params if params else dict()
self.estimators = [LGBMRegressor(**params)for _ in range(n_estimators)]
self.max_sample = max_sample
def fit(self, X, y):
scores = []
for estimator in self.estimators:
idx = X.sample(frac=self.max_sample, replace=False ).index
X_train, y_train = X.loc[idx], y.loc[idx]
X_test, y_test = X[~X.index.isin(idx)], y[~y.index.isin(idx)]
estimator.fit(X_train,
y_train,
eval_set =(X_test,
y_test),
eval_metric='rmse',
early_stopping_rounds=500,
verbose=False 
)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 return scores
def predict(self, X):
pred = np.zeros(X.shape[0])
 for estimator in self.estimators:
pred += estimator.predict(X)/ len(self.estimators)
 return pred<train_model>
def prepare_image(datadir, featherdir, data_type='train',
submission=False, indices=[0, 1, 2, 3]):
assert data_type in ['train', 'test']
if submission:
image_df_list = [pd.read_parquet(datadir / f'{data_type}_image_data_{i}.parquet')
 for i in indices]
else:
image_df_list = [pd.read_feather(featherdir / f'{data_type}_image_data_{i}.feather')
 for i in indices]
print('image_df_list', len(image_df_list))
HEIGHT = 137
WIDTH = 236
images = [df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH)for df in image_df_list]
del image_df_list
gc.collect()
images = np.concatenate(images, axis=0)
 return images
params = {
'learning_rate': 0.005,
'n_estimators': 10000,
'num_leaves': 511,
'feature_fraction': 0.4,
'bagging_freq': 2,
'bagging_fraction': 0.8,
'lambda_l1': 10,
'seed': 2021
}
model = BaggingLGBM(n_estimators=20, max_sample=0.85, params=params)
 scores = model.fit(train[predictors], train[target])
 np.mean(scores )<save_to_csv>
def predict_core(test_images, image_size, threshold,
arch, n_total, model_name, load_model_path, batch_size=512, device='cuda:0', **kwargs):
classifier = build_classifier(arch, load_model_path, n_total, model_name, device=device)
 test_dataset = BengaliAIDataset(
 test_images, None,
transform=Transform(affine=False, crop=True, size=(image_size, image_size),
threshold=threshold, train=False))
print('test_dataset', len(test_dataset))
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 test_pred_proba = classifier.predict_proba(test_loader)
 return test_pred_proba
test[target] = model.predict(test[predictors])
 test[target].to_csv('submission.csv' )<install_modules>
device = torch.device(device)
 n_grapheme = 168
n_vowel = 11
n_consonant = 7
n_total = n_grapheme + n_vowel + n_consonant
print('n_total', n_total )
system('pip install pycaret');<import_modules>
traindir = '/kaggle/input/bengaliaicv19-trainedmodels/'
data_type = 'test'
test_preds_list = []
for i in range(4):
indices = [i]
test_images = prepare_image(
 datadir, featherdir, data_type=data_type, submission=submission, indices=indices)
 n_dataset = len(test_images)
 print(f'n_dataset={n_dataset}')
 model_preds_list = []
for j in range(4):
train_args_dict = load_json(os.path.join(traindir, f'args_{j}.json'))
train_args_dict.update({
'load_model_path': os.path.join(traindir, f'predictor_{j}.pt'),
'device': device,
'batch_size': batch_size,
'debug': debug,
})
 print(f'j {j} updated train_args_dict {train_args_dict}')
 test_preds = predict_core(
 test_images=test_images, n_total=n_total,
**train_args_dict)
 model_preds_list.append(test_preds)
 proba0 = torch.mean(torch.stack([test_preds[0] for test_preds in model_preds_list], dim=0), dim=0)
 proba1 = torch.mean(torch.stack([test_preds[1] for test_preds in model_preds_list], dim=0), dim=0)
 proba2 = torch.mean(torch.stack([test_preds[2] for test_preds in model_preds_list], dim=0), dim=0)
 p0 = torch.argmax(proba0, dim=1 ).cpu().numpy()
p1 = torch.argmax(proba1, dim=1 ).cpu().numpy()
p2 = torch.argmax(proba2, dim=1 ).cpu().numpy()
print('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 test_preds_list.append([p0, p1, p2])
 if debug:
break
del test_images
gc.collect()
import pandas as pd
from pycaret.regression import *<load_from_csv>
p0 = np.concatenate([test_preds[0] for test_preds in test_preds_list], axis=0)
 p1 = np.concatenate([test_preds[1] for test_preds in test_preds_list], axis=0)
 p2 = np.concatenate([test_preds[2] for test_preds in test_preds_list], axis=0)
 print('concat:', 'p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 row_id = []
target = []
for i in tqdm(range(len(p0))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [p0[i], p1[i], p2[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False)
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv', index_col='id')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv', index_col='id' )<train_model>
train = pd.read_csv(datadir/'train.csv' )
setup(data=train,
target='target',
fold=4,
silent=True,
verbose=False,
session_id=2021);<choose_model_class>
pred_df = pd.DataFrame({
'grapheme_root': p0,
'vowel_diacritic': p1,
'consonant_diacritic': p2
} )
model = create_model('lightgbm' )<train_model>
train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values