Kaggle competition: Kannada MNIST
top_features = [i for i in eli5.formatters.as_dataframe.explain_weights_df(model ).feature if 'BIAS' not in i][:100]
X1 = X[top_features]
X_train, X_valid, y_train, y_valid = train_test_split(X1, y, test_size=0.2, stratify=y)
 model.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_valid, y_valid)], verbose=1000, early_stopping_rounds=200 )<compute_train_metric>
def random_seed(seed_value, use_cuda):
np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 random.seed(seed_value)
 if use_cuda: torch.cuda.manual_seed_all(seed_value)
 random_seed(42, True )
def calculate_metrics(model, X_train: pd.DataFrame() = None, y_train: pd.DataFrame() = None, X_valid: pd.DataFrame() = None,
y_valid: pd.DataFrame() = None, columns: list = [])-> pd.DataFrame() :
columns = columns if len(columns)> 0 else list(X_train.columns)
 train_pred = model.predict_proba(X_train[columns])
 valid_pred = model.predict_proba(X_valid[columns])
 f1 = 0
best_t = 0
for t in np.arange(0.1, 1, 0.05):
valid_pr =(valid_pred[:, 1] > t ).astype(int)
 valid_f1 = metrics.f1_score(y_valid, valid_pr)
 if valid_f1 > f1:
f1 = valid_f1
best_t = t
t = best_t
train_pr =(train_pred[:, 1] > t ).astype(int)
 valid_pr =(valid_pred[:, 1] > t ).astype(int)
 train_f1 = metrics.f1_score(y_train, train_pr)
 valid_f1 = metrics.f1_score(y_valid, valid_pr)
 score_df = []
print(f'Best threshold: {t:.2f}.Train f1: {train_f1:.4f}.Valid f1: {valid_f1:.4f}.')
 score_df.append(['F1', np.round(train_f1, 4), np.round(valid_f1, 4)])
 train_r = metrics.recall_score(y_train, train_pr)
 valid_r = metrics.recall_score(y_valid, valid_pr)
 score_df.append(['Recall', np.round(train_r, 4), np.round(valid_r, 4)])
 train_p = metrics.precision_score(y_train, train_pr)
 valid_p = metrics.precision_score(y_valid, valid_pr)
 score_df.append(['Precision', np.round(train_p, 4), np.round(valid_p, 4)])
 train_roc = metrics.roc_auc_score(y_train, train_pred[:, 1])
 valid_roc = metrics.roc_auc_score(y_valid, valid_pred[:, 1])
 score_df.append(['ROCAUC', np.round(train_roc, 4), np.round(valid_roc, 4)])
 train_apc = metrics.average_precision_score(y_train, train_pred[:, 1])
 valid_apc = metrics.average_precision_score(y_valid, valid_pred[:, 1])
 score_df.append(['APC', np.round(train_apc, 4), np.round(valid_apc, 4)])
 print(metrics.confusion_matrix(y_valid, valid_pr))
score_df = pd.DataFrame(score_df, columns=['Metric', 'Train', 'Valid'])
 print(score_df)
 return score_df, t<compute_test_metric>
train_csv = pd.read_csv(PATH/'train.csv')
 train_csv.head().T
_ = calculate_metrics(model, X_train, y_train, X_valid, y_valid )<feature_engineering>
def get_data_labels(csv,label):
fileraw = pd.read_csv(csv)
 labels = fileraw[label].to_numpy()
data = fileraw.drop([label],axis=1 ).to_numpy(dtype=np.float32 ).reshape(( fileraw.shape[0],28,28))
data = np.expand_dims(data, axis=1)
 return data, labels
train_data, train_labels = get_data_labels(PATH/'train.csv','label')
 test_data, test_labels = get_data_labels(PATH/'test.csv','id')
 other_data, other_labels = get_data_labels(PATH/'Dig-MNIST.csv','label' )
<normalization>
plt.title(f'Training Label: {train_labels[43]}')
 plt.imshow(train_data[43,0],cmap='gray');
%%time
X = train.drop(['ID_code', 'target'], axis=1)
 X_test = test.drop(['ID_code'], axis=1)
 neigh = NearestNeighbors(3, n_jobs=-1)
 neigh.fit(X)
 dists, _ = neigh.kneighbors(X, n_neighbors=3)
 mean_dist = dists.mean(axis=1)
 max_dist = dists.max(axis=1)
 min_dist = dists.min(axis=1)
 X['mean_dist'] = mean_dist
X['max_dist'] = max_dist
X['min_dist'] = min_dist
test_dists, _ = neigh.kneighbors(X_test, n_neighbors=3)
 test_mean_dist = test_dists.mean(axis=1)
 test_max_dist = test_dists.max(axis=1)
 test_min_dist = test_dists.min(axis=1)
 X_test['mean_dist'] = test_mean_dist
X_test['max_dist'] = test_max_dist
X_test['min_dist'] = test_min_dist
oof_lgb, prediction_lgb_dist, scores = train_model(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = prediction_lgb_dist
sub.to_csv('lgb_dist.csv', index=False )<save_to_csv>
np.random.seed(42)
 ran_20_pct_idx =(np.random.random_sample(train_labels.shape)) <.2
train_80_labels = train_labels[np.invert(ran_20_pct_idx)]
train_80_data = train_data[np.invert(ran_20_pct_idx)]
valid_20_labels = train_labels[ran_20_pct_idx]
valid_20_data = train_data[ran_20_pct_idx]
<import_modules>
train_ds = ArrayDataset(train_80_data,train_80_labels)
 valid_ds = ArrayDataset(valid_20_data,valid_20_labels)
 other_ds = ArrayDataset(other_data, other_labels)
 test_ds = ArrayDataset(test_data, test_labels )
import pandas as pd
import numpy as np
from os import listdir
from os.path import isfile, join
from IPython.display import display
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
import warnings
import random as rnd
from sklearn import preprocessing
from sklearn.cluster import MiniBatchKMeans
from sklearn.decomposition import PCA
from sklearn.model_selection import KFold
import xgboost as xgb
import lightgbm as lgb
from sklearn import metrics
from sklearn.model_selection import train_test_split
import scipy
import matplotlib.pyplot as plt
from scipy.cluster import hierarchy as hc<define_variables>
bs = 64
data = DataBunch.create(train_ds, valid_ds, test_ds=test_ds, bs=bs )
path =".. /input"<define_variables>
MODEL_DIR = Path('.. /working/models/' )
allfiles = [f for f in listdir(path)if isfile(join(path, f)) if(not f.endswith('.ipynb')) ]
allfiles<train_model>
class ConvNet1(nn.Module):
def __init__(self):
super(ConvNet1, self ).__init__()
self.layer1 = nn.Sequential(
 nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(32)
)
 self.layer2 = nn.Sequential(
 nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(32)
)
 self.layer3 = nn.Sequential(
 nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, padding=1, stride=2),
nn.ReLU() ,
nn.BatchNorm2d(64)
)
 self.layer4 = nn.Sequential(
 nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(64)
)
 self.layer5 = nn.Sequential(
 nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(64)
)
 self.layer6 = nn.Sequential(
 nn.Conv2d(in_channels=64, out_channels=128, kernel_size=5, padding=1, stride=2),
nn.ReLU() ,
nn.BatchNorm2d(128), 
)
 self.drop_out = nn.Dropout(0.2)
 self.relu = nn.ReLU()
self.fc1 = nn.Linear(4608, 128)
 self.fc2 = nn.Linear(128, 10)
 self.bn1d = nn.BatchNorm1d(128)
 self.output = nn.Softmax(dim=1)
 def forward(self, x):
out = self.layer1(x)
 out = self.layer2(out)
 out = self.drop_out(self.layer3(out))
out = self.layer4(out)
 out = self.layer5(out)
 out = self.drop_out(self.layer6(out))
out = out.view(out.shape[0], -1)
 out = self.fc1(out)
 out = self.relu(out)
 out = self.bn1d(out)
 out = self.drop_out(out)
 out = self.fc2(out)
 out = self.output(out)
 return out
def print_size(file):
statinfo=os.stat(join(path,file))
print('Size of file {} in Bytes is {} bytes'.format(str(file),str(statinfo.st_size)) )<load_from_csv>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 conv_net_1 = ConvNet1()
conv_net_1 = conv_net_1.to(device )
def load_df(df,**kwds):
if str(df ).endswith('.csv'):
ndf =pd.read_csv(str(df),**kwds)
 else:
print('Could not find the df')
 print('Loaded file {} with shape{}'.format(df,ndf.shape))
return ndf<load_from_csv>
class ConvNet2(nn.Module):
def __init__(self):
super(ConvNet2, self ).__init__()
self.layer1 = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=5, stride=1, padding=2),
nn.ReLU() ,
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer2 = nn.Sequential(
 nn.Conv2d(32, 64, kernel_size=5, stride=1, padding=2),
nn.ReLU() ,
nn.MaxPool2d(kernel_size=2, stride=2))
self.drop_out = nn.Dropout(0.5)
 self.fc1 = nn.Linear(7 * 7 * 64, 1000)
 self.fc2 = nn.Linear(1000, 10)
 self.relu = nn.ReLU()
def forward(self, x):
out = self.layer1(x)
 out = self.layer2(out)
 out = out.reshape(out.size(0), -1)
 out = self.drop_out(out)
 out = self.fc1(out)
 out = self.relu(out)
 out = self.drop_out(out)
 out = self.fc2(out)
 return out
test =load_df(join(path,allfiles[0]))
train =load_df(join(path,allfiles[1]))
submission =load_df(join(path,allfiles[2]))<count_values>
conv_net_2 = ConvNet2()
conv_net_2 = conv_net_2.to(device )
train.target.value_counts()<drop_column>
def conv2(ni,nf,stride=2,ks=3): return conv_layer(ni,nf,stride=stride,ks=ks )
train_df =train.drop(['ID_code','target'],axis =1 )<count_missing_values>
conv_net_3 = nn.Sequential(
 conv2(1,32,stride=1,ks=3),
conv2(32,32,stride=1,ks=3),
conv2(32,32,stride=2,ks=5),
nn.Dropout(0.4),
conv2(32,64,stride=1,ks=3),
conv2(64,64,stride=1,ks=3),
conv2(64,64,stride=2,ks=5),
nn.Dropout(0.4),
Flatten() ,
nn.Linear(3136, 128),
relu(inplace=True),
nn.BatchNorm1d(128),
nn.Dropout(0.4),
nn.Linear(128,10)
 )
train_df.isnull().sum().sort_index() /len(train_df )<compute_test_metric>
rn18 = models.resnet18(pretrained=False)
 rn18.load_state_dict(torch.load('.. /input/pytorch-pretrained-models/resnet18-5c106cde.pth'))
rn18.conv1 = Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False )
def auc(x,y): return metrics.roc_auc_score(x,y)
 def print_score(m):
res = [auc(y_train,m.predict_proba(X_train)[:,1]),auc(y_valid,m.predict_proba(X_valid)[:,1]),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
print(res )<split>
learner1 = Learner(data,
conv_net_1,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=Adam,
loss_func=nn.CrossEntropyLoss()
)
df_trn, y_trn =train_df, train['target']
df_tst = test.drop(['ID_code'], axis=1)
 X_train, X_valid,y_train, y_valid = train_test_split(df_trn,y_trn, test_size =.20, stratify =y_trn,random_state =2019)
 X_train.shape, y_train.shape, X_valid.shape<train_model>
learner1.lr_find()
learner1.recorder.plot(suggestion=True )
m = RandomForestClassifier(n_estimators=10, n_jobs=8,min_samples_leaf=3, max_features=0.5)
 m.fit(X_train, y_train)
 print_score(m )<compute_test_metric>
%%time
learner1.fit_one_cycle(50,
slice(1e-03),
callbacks=[SaveModelCallback(learner1,
every='improvement',
monitor='accuracy',
name='best_model_1')]
)
fi = rf_feat_importance(m, df_trn); fi[:30]<features_selection>
learner1.load('best_model_1')
 preds, ids = learner1.get_preds(DatasetType.Test)
 y = torch.argmax(preds, dim=1)
 submission_1 = pd.DataFrame({ 'id': ids,'label': y })
 submission_1.to_csv("submission.csv", index=False)
 submission_1.to_csv("submission_1.csv", index=False )
to_keep = fi[fi.imp>0.005].cols; len(to_keep )<split>
learner2 = Learner(data,
conv_net_2,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=RAdam,
loss_func=nn.CrossEntropyLoss()
)
df_keep = df_trn[to_keep].copy()
X_train, X_valid,y_train, y_valid = train_test_split(df_trn,y_trn, test_size =.20, stratify =y_trn,random_state =2019 )<find_best_model_class>
learner2.lr_find()
learner2.recorder.plot(suggestion=True )
class Model(object):
def __init__(self,trn,tst,tget,subm):
self.trn = trn
self.tst = tst
self.tget = tget
self.subm =subm
def predict(self, path, model):
X_train, X_val, y_train, y_val = self.kfold_cv()
y_pred = model(X_train, y_train, X_val, y_val)
 self.save_pred(y_pred, path)
 def kfold_cv(self):
X = self.trn.values
y = self.tget.values
kf = KFold(n_splits=5, random_state=2019)
 kf.get_n_splits(X)
 for train_index, val_index in kf.split(X):
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
return X_train, X_val, y_train, y_val
def xgb(self, X_train, y_train, X_val, y_val):
dtrain = xgb.DMatrix(X_train, label = y_train)
 dvalid = xgb.DMatrix(X_val, label = y_val)
 dtest = xgb.DMatrix(self.tst.values)
 dlist = [(dtrain, 'train'),(dvalid, 'valid')]
parameters = {'eta': 0.025,
'colsample_bytree': 0.7,
'max_depth': 4,
'subsample': 0.7,
'nthread': -1,
'booster' : 'gbtree',
'colsample_bytree': 0.7,
'min_child_weight': 50,
'colsample_bylevel': 0.7,
'lambda' :1,
'alpha':0,
'eval_metric' : "auc",
'silent': 1,
'objective': 'binary:logistic'}
model = xgb.train(parameters, dtrain, 20000, dlist, early_stopping_rounds=300,
verbose_eval=100)
 print("Start prediction...")
 y_pred = model.predict(dtest)
 return y_pred
def lgb(self, X_train, y_train, X_val, y_val):
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_val, label=y_val)
 dtest = self.tst.values
dlist = [dtrain, dvalid]
parameters = {'learning_rate': 0.025,
'colsample_bytree': 0.3,
'max_depth': 6,
'subsample': 0.9,
'num_threads': -1,
'boosting_type' : 'gbdt',
'metric' : 'auc',
'objective': 'binary',
'num_leaves' : 20,
'sub_feature':0.7,
'sub_row' :0.7,
'bagging_freq':1,
'lambda_l1':5,
'lambda_l2': 5}
model = lgb.train(parameters, dtrain, 20000, dlist,early_stopping_rounds=300, verbose_eval=100)
 print("Start prediction...")
 y_pred = model.predict(dtest)
 return y_pred
def save_pred(self, y_pred, path):
self.subm['target'] = y_pred
self.subm[['ID_code','target']].to_csv(path, index=False )<train_model>
%%time
learner2.fit_one_cycle(50,
slice(1e-03),
callbacks=[SaveModelCallback(learner2,
every='improvement',
monitor='accuracy',
name='best_model_2')]
)
print("Training model...
 ")
 model = Model(df_trn,df_tst,y_trn,submission)
 y_pred = model.predict(path ='submission_lgb.csv', model=model.lgb )<train_model>
learner2.load('best_model_2')
 preds, ids = learner2.get_preds(DatasetType.Test)
 y = torch.argmax(preds, dim=1)
 submission_2 = pd.DataFrame({ 'id': ids,'label': y })
 submission_2.to_csv("submission_2.csv", index=False )
print("Training model...
 ")
 model = Model(df_trn,df_tst,y_trn,submission)
 y_pred = model.predict(path ='submission_xgb.csv', model=model.xgb )<import_modules>
learner3 = Learner(data,
conv_net_3,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=Adam,
loss_func=nn.CrossEntropyLoss()
)
import pandas as pd
from pathlib import Path<define_variables>
learner3.lr_find()
learner3.recorder.plot(suggestion=True )
root = Path(".. /input" )<load_from_csv>
%%time
learner3.fit_one_cycle(50,
slice(8e-03),
callbacks=[SaveModelCallback(learner3,
every='improvement',
monitor='accuracy',
name='best_model_3')]
)
train = pd.read_csv(root.joinpath("train.csv"))
test = pd.read_csv(root.joinpath("test.csv"))<drop_column>
learner3.load('best_model_3')
 preds, ids = learner3.get_preds(DatasetType.Test)
 y = torch.argmax(torch.exp(preds), dim=1)
 submission_3 = pd.DataFrame({ 'id': ids,'label': y })
 submission_3.to_csv("submission_3.csv", index=False )
train_id = train.ID_code
test_id = test.ID_code
target = train.target
train.drop(columns=["ID_code", "target"], inplace=True)
 test.drop(columns=["ID_code"], inplace=True )<import_modules>
learner4 = Learner(data,
rn18,
metrics=accuracy,
model_dir=MODEL_DIR,
opt_func=Adam,
loss_func=nn.CrossEntropyLoss()
)
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
from catboost import Pool, CatBoostClassifier<choose_model_class>
learner4.lr_find()
learner4.recorder.plot(suggestion=True )
model = CatBoostClassifier(loss_function="Logloss",
eval_metric="AUC",
task_type="GPU",
learning_rate=0.01,
iterations=10000,
random_seed=42,
od_type="Iter",
depth=10,
early_stopping_rounds=500
)<choose_model_class>
%%time
learner4.fit_one_cycle(50,
slice(1e-03),
callbacks=[SaveModelCallback(learner4,
every='improvement',
monitor='accuracy',
name='best_model_4')]
)
n_split = 5
kf = KFold(n_splits=n_split, random_state=42, shuffle=True )<prepare_x_and_y>
learner4.load('best_model_4')
 preds, ids = learner4.get_preds(DatasetType.Test)
 y = torch.argmax(torch.exp(preds), dim=1)
 submission_4 = pd.DataFrame({ 'id': ids,'label': y })
 submission_4.to_csv("submission_4.csv", index=False )
y_valid_pred = 0 * target
y_test_pred = 0<train_model>
flatten = lambda l: [np.float32(item)for sublist in l for item in sublist]
metrics_list_1 = flatten(learner1.recorder.metrics)
 metrics_list_2 = flatten(learner2.recorder.metrics)
 metrics_list_3 = flatten(learner3.recorder.metrics)
 metrics_list_4 = flatten(learner4.recorder.metrics )
<save_to_csv><EOS> 
blended_preds = np.round(( submission_1['label'] + submission_2['label'] +
submission_3['label'] + submission_4['label'])/4)
 blended_submission = pd.DataFrame({'id': ids, 'label': blended_preds})
 blended_submission['label'] = blended_submission['label'].astype(np.uint8)
 blended_submission.to_csv("blended_submission.csv", index=False )