Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<load_from_csv>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Input, Dense, LeakyReLU, Flatten, Dropout, BatchNormalization,Add,GlobalAveragePooling2D,Softmax
from tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint,EarlyStopping
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import math
from sklearn.model_selection import train_test_split
import random
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 remove_vehicle = True
if remove_vehicle:
train_features = train.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 else:
train_features = train 
<count_values>
train_datas = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 val_datas = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv" )
col_features = list(train_features.columns)[1:]
print(train_features[col_features[0]].value_counts())
 print(test_features[col_features[0]].value_counts())
 print(train_features[col_features[1]].value_counts())
 print(test_features[col_features[1]].value_counts())
 print(train_features[col_features[2]].value_counts())
 print(test_features[col_features[2]].value_counts())
<count_missing_values>
print(train_datas.shape)
 print(val_datas.shape)
 datas = pd.concat([train_datas,val_datas],axis=0)
 print(datas.shape)
 datas_X = np.array(datas.drop("label",axis=1),dtype=np.float32)
 datas_Y = np.array(datas[["label"]],dtype=np.int32)
 train_X,val_X,train_Y,val_Y = train_test_split(datas_X,datas_Y,test_size=0.1,shuffle=True )
print(test_features[col_features].isna().sum().values.sum())
 print(np.isinf(test_features[col_features[3:]].values ).sum())
 print(train_features[col_features].isna().values.sum())
 print(np.isinf(train_features[col_features[3:]].values ).sum() )<data_type_conversions>
tf.reset_default_graph()
nets = 1
model = [0]*nets
for j in range(nets):
input_layer = Input(shape=(28,28,1))
net = Conv2D(64,(3,3), padding='same' )(input_layer)
 net = BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = Conv2D(64,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = MaxPooling2D(2, 2 )(net)
 net = Conv2D(128,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = Conv2D(128,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = MaxPooling2D(2,2 )(net)
 net = Dropout(0.2 )(net)
 net = Conv2D(256,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = Conv2D(256,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = MaxPooling2D(2,2 )(net)
 net = Dropout(0.2 )(net)
 net = GlobalAveragePooling2D()(net)
 net = BatchNormalization()(net)
 output = Dense(10, activation='softmax' )(net)
 model[j] = Model(inputs=input_layer,outputs=output)
 model[j].compile(optimizer='Adam', loss="sparse_categorical_crossentropy",metrics=["accuracy"])
 print(model[0].summary() )
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 print(ratios)
<normalization>
datagen = ImageDataGenerator(
 rotation_range = 20,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 0.1,
zoom_range = 0.25,
horizontal_flip = False,
rescale=1/255.0 )
columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored)
 print(ratios_nonscored)
 print(len(columns_nonscored), len(ratios_nonscored))<categorify>
annealer = LearningRateScheduler(lambda x: 0.002 * 0.99 ** x)
 model_checks = []
for j in range(nets):
model_checks.append(ModelCheckpoint("model-{}.ckpt".format(j), monitor='val_acc', verbose=1, save_best_only=True,))
early_stop = EarlyStopping(monitor='val_acc', verbose=2, patience=30 )
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features.columns)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32)
<categorify>
history = [0] * nets
epochs = 150
batch_size = 1024
for j in range(nets):
history[j] = model[j].fit_generator(datagen.flow(train_X,train_Y, batch_size=batch_size),
epochs = epochs,
steps_per_epoch = math.ceil(train_X.shape[0]/batch_size)*2,
validation_data =(val_X/255.0,val_Y),
callbacks=[annealer,model_checks[j],early_stop],
verbose=2)
 print("Train: Model{:d}: Epochs={:d},LearningRate={:.12f} ,Train accuracy={:.5f}, Validation accuracy={:.5f}".format(
 j+1,epochs,tf.keras.backend.get_value(model[j].optimizer.lr),max(history[j].history['acc']),max(history[j].history['val_acc'])) )
class MOADataset(Dataset):
def __init__(self, x_cats, x_nums, y=None, y2=None):
self.cats = x_cats
self.nums = x_nums
self.y = y
self.y2 = y2
def __len__(self):
return len(self.cats)
 def __getitem__(self, index):
x1 = torch.as_tensor(self.cats[index], dtype=torch.long)
 x2 = torch.as_tensor(self.nums[index], dtype=torch.float)
 if self.y is not None:
label = torch.as_tensor(self.y[index], dtype=torch.float)
 if self.y2 is not None:
label2 = torch.as_tensor(self.y2[index], dtype=torch.float)
 return x1, x2, label, label2
return x1, x2, label
return x1, x2<choose_model_class>
test_csv = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 data_test = np.array(test_csv.drop("id",axis=1),dtype=np.float32)
 data_test = np.reshape(data_test,(-1,28,28,1))
X_test = data_test / 255.0
print(X_test.shape,X_test.dtype )
class MOA_MLP(nn.Module):
def __init__(self, num_cats=[2,3,2] , cats_emb_size=[2,2,2], num_numericals=872, hidden_size_numericals=2048,
num_class=206, aux=None):
super().__init__()
self.cat_emb1 = nn.Embedding(num_cats[0], cats_emb_size[0], padding_idx=0)
 self.cat_emb2 = nn.Embedding(num_cats[1], cats_emb_size[1], padding_idx=0)
 self.norms = nn.BatchNorm1d(sum(cats_emb_size)+num_numericals)
 self.dropout = nn.Dropout(0.2)
 self.proj = nn.utils.weight_norm(nn.Linear(sum(cats_emb_size)+ num_numericals, hidden_size_numericals))
self.norm_proj = nn.BatchNorm1d(hidden_size_numericals)
 self.dropout2 = nn.Dropout(0.5)
 hd_1 = hidden_size_numericals//2
hd_2 = hd_1//2
self.extractor = nn.Sequential(nn.utils.weight_norm(nn.Linear(hidden_size_numericals, hd_1)) ,
nn.PReLU() ,
nn.BatchNorm1d(hd_1),
nn.Dropout(0.5), 
)
 self.cls = nn.utils.weight_norm(nn.Linear(hd_1, num_class))
self.cls_aux=None
if aux is not None:
self.cls_aux = nn.utils.weight_norm(nn.Linear(hd_1, aux))
def forward(self, x_cat, x_num):
cat_features = torch.cat([self.cat_emb1(x_cat[:,0]), self.cat_emb2(x_cat[:,1])], dim=1)
 all_features = torch.cat([cat_features, x_num], dim=1)
 all_features = self.norms(all_features)
 all_features = self.dropout(all_features)
 proj_features = self.proj(all_features)
 proj_features = self.norm_proj(F.relu(proj_features))
proj_features = self.dropout2(proj_features)
 features_reduced = self.extractor(proj_features)
 outputs = self.cls(features_reduced)
 if self.cls_aux is not None:
outputs2 = self.cls_aux(features_reduced)
 return outputs, outputs2
return outputs<normalization>
for j in range(nets):
model[j].load_weights("model-{}.ckpt".format(j))
def train_one_epoch(model, dataloader, cfg, optimizer, loss_fn, loss_fn_aux=None, accumulation=1, with_aux_class=False, verbose=True):
model.train()
scaler = GradScaler()
optimizer.zero_grad()
N = 0.
 total_loss = 0.
 t=tqdm(dataloader, disable=~verbose)
 for i, batch in enumerate(t):
x1 = batch[0]
x2 = batch[1]
labels = batch[2]
x1 = x1.to(cfg.device)
 x2 = x2.to(cfg.device)
 labels = labels.to(cfg.device)
 if with_aux_class:
labels2 = batch[3]
labels2 = labels2.to(cfg.device)
 with autocast(cfg.use_apex):
if with_aux_class:
outputs, outputs2 = model(x1, x2)
 loss1 = loss_fn(outputs, labels ).mean(0 ).mean()
loss2 = loss_fn_aux(outputs2, labels2 ).mean(0 ).mean()
loss = loss1 + 0.5*loss2
else:
outputs = model(x1, x2)
 loss = loss_fn(outputs, labels ).mean(0 ).mean()
N += len(x1)
 total_loss +=(loss.item() * len(x1))
if cfg.use_apex:
loss = loss/accumulation
scaler.scale(loss ).backward()
else:
loss = loss/accumulation
loss.backward()
if(i+1)%accumulation == 0 or i-1 == len(dataloader):
if cfg.use_apex:
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
else:
optimizer.step()
optimizer.zero_grad()
t.set_description("Loss : {0}".format(total_loss/N))
t.refresh()
def evals(model, dataloader, cfg, loss_fn, loss_fn_aux=None, verbose=True):
model.eval()
N = 0.
 total_loss = 0.
 y_preds = []
y_targets = []
t=tqdm(dataloader, disable=~verbose)
 with torch.no_grad() :
for i, batch in enumerate(t):
x1 = batch[0]
x2 = batch[1]
labels = batch[2]
x1 = x1.to(cfg.device)
 x2 = x2.to(cfg.device)
 labels = labels.to(cfg.device)
 if cfg.with_aux_class:
labels2 = batch[3]
labels2 = labels2.to(cfg.device)
 with autocast(cfg.use_apex):
if cfg.with_aux_class:
outputs, outputs2 = model(x1, x2)
 loss1 = loss_fn(outputs, labels ).mean(0 ).mean()
loss2 = loss_fn_aux(outputs2, labels2 ).mean(0 ).mean()
loss = loss1 + 0.5*loss2
else:
outputs = model(x1, x2)
 loss = loss_fn(outputs, labels ).mean(0 ).mean()
N += len(x1)
 total_loss +=(loss.item() * len(x1))
t.set_description("Loss : {0}".format(total_loss/N))
t.refresh()
y_preds.append(torch.sigmoid(outputs ).detach().cpu().numpy())
 y_targets.append(labels.detach().cpu().numpy())
 y_preds = np.concatenate(y_preds, axis=0)
 y_targets = np.concatenate(y_targets, axis=0)
 score = log_loss_multi(y_targets, y_preds)
 return y_preds, y_targets, score
def inference_fn(model, dataloader, cfg,verbose=True):
model.eval()
N = 0.
 y_preds = []
with torch.no_grad() :
for i, batch in enumerate(tqdm(dataloader, disable=~verbose)) :
x1 = batch[0]
x2 = batch[1]
x1 = x1.to(cfg.device)
 x2 = x2.to(cfg.device)
 with autocast(cfg.use_apex):
if cfg.with_aux_class:
outputs, outputs2 = model(x1, x2)
 else:
outputs = model(x1, x2)
 y_preds.append(torch.sigmoid(outputs ).detach().cpu().numpy())
 y_preds = np.concatenate(y_preds, axis=0)
 return y_preds<compute_test_metric>
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1 )
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()<feature_engineering>
submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 submission['label'] = results
submission.to_csv("submission.csv",index=False )