Kaggle competition: Kannada MNIST
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit["target"] = x_test_second_layer.mean(1)
 submit.to_csv("submission.csv", index=False )<set_options>
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Activation,BatchNormalization,Conv2D,Dense,Dropout,Flatten,MaxPooling2D
from tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint,EarlyStopping
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
np.random.seed(0 )<load_from_csv>
train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv")
 Dig_MNIST = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv")
 sample_sub = pd.read_csv("/kaggle/input/Kannada-MNIST/sample_submission.csv" )
train = pd.read_csv('.. /input/train.csv', index_col='id')
 test = pd.read_csv('.. /input/test.csv', index_col='id')
 data = pd.concat([train, test], sort=False )<prepare_x_and_y>
X=train.iloc[:,1:].values
Y=train.iloc[:,0].values
Y[:10]
methods = ['diff', 'discrete']
methods += ['random'+str(i)for i in range(1, 100)]
pred = pd.DataFrame(index=test.index, columns=methods)
 for i in range(512):
cur = data[data['wheezy-copper-turtle-magic']==i]
train_idx = [i for i in cur.index if i in train.index]
test_idx = [i for i in cur.index if i in test.index]
y_data = cur['target']
x_data = cur.loc[:, cur.std() >2]
cur_test = x_data.loc[test_idx]
gm = GaussianMixture(6, n_init=20, max_iter=10000, tol=1e-5 ).fit(x_data)
 tmp = pd.DataFrame([gm.predict(x_data), y_data.values] ).dropna(axis=1)
 pos_val_counts = tmp.loc[:, tmp.loc[1]==1].loc[0].value_counts()
neg_val_counts = tmp.loc[:, tmp.loc[1]==0].loc[0].value_counts()
pos_cluster = pos_val_counts.iloc[:3].index.astype(int)
 neg_cluster = neg_val_counts.iloc[:3].index.astype(int)
 pos_mns = [multivariate_normal(gm.means_[i], gm.covariances_[i])for i in pos_cluster]
neg_mns = [multivariate_normal(gm.means_[i], gm.covariances_[i])for i in neg_cluster]
pos_pred = np.max([pos_mn.logpdf(cur_test)for pos_mn in pos_mns], axis=0)
 neg_pred = np.max([neg_mn.logpdf(cur_test)for neg_mn in neg_mns], axis=0)
 pred.loc[cur_test.index, 'diff'] = pos_pred - neg_pred
pred.loc[cur_test.index, 'discrete'] = pd.Series(gm.predict(cur_test)).replace(pos_cluster, -1 ).replace(neg_cluster, 0 ).abs().values
for i in range(1, 100):
pred['random'+str(i)] = pred['discrete']
ascending = np.arange(2, len(pred)+2)
 n_zeros =(pred['discrete']==0 ).sum()
n_ones =(pred['discrete']==1 ).sum()
zeros = ascending[:n_zeros].copy()
ones = ascending[-n_ones:].copy()
np.random.shuffle(zeros)
 np.random.shuffle(ones)
 pred['random'+str(i)][pred['discrete']==0] = zeros
pred['random'+str(i)][pred['discrete']==1] = ones<save_to_csv>
Y = tf.keras.utils.to_categorical(Y, num_classes=10)
 print(f"Y data shape: {Y.shape}" )
for i in methods:
sub = pd.DataFrame({'id': pred.index, 'target': pred[i].values})
 sub.to_csv(i+'.csv', index=False)
 sub.head(20 )<import_modules>
X_train, X_valid, Y_train, Y_valid = train_test_split(X, Y, test_size = 0.2, random_state=42 )
import numpy as np
import pandas as pd
from sklearn.covariance import OAS
from sklearn.metrics import roc_auc_score
from sklearn.feature_selection import VarianceThreshold
from sklearn.cluster import KMeans
from sklearn.mixture import GaussianMixture<compute_train_metric>
train_datagen = ImageDataGenerator(rescale = 1./255,
rotation_range = 10,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 0.1,
zoom_range = 0.25,
horizontal_flip=False )
def MPEstimate(X, y, seed, n_clusters_per_class=3):
emu, ep = [], []
for yy in [0, 1]:
XX = X[y==yy]
ca = KMeans(n_clusters=n_clusters_per_class, random_state=seed)
 cluster = ca.fit_predict(XX)
 for i in range(n_clusters_per_class):
C = XX[cluster==i]
o = OAS().fit(C)
 mu = o.location_
p = o.precision_
emu.append(mu)
 ep.append(p)
 return np.stack(emu), np.stack(ep )<compute_test_metric>
valid_datagen = ImageDataGenerator(rescale=1./255 )
def betterAucOrdering(data, preds, flips, eps = 1e-12, ascending=True):
preds2 = preds.copy()
rank = pd.Series(flips ).rank(method='dense', ascending=ascending)
 for(r, g)in zip(rank, groups):
idx = data[group]==g
p = preds[idx].values
p[p<0.5] += eps*r
p[p>0.5] -= eps*r
preds2[idx] = p
return preds2<load_from_csv>
model = Sequential([
Conv2D(64, 3, padding='same', input_shape=(28, 28, 1)) ,
BatchNormalization(scale=False, center=True),
Activation('relu'),
Conv2D(64, 3, padding='same'),
BatchNormalization(scale=False, center=True),
Activation('relu'),
MaxPooling2D(2,2),
Dropout(0.2),
Conv2D(128, 3, padding='same'),
BatchNormalization(scale=False, center=True),
Activation('relu'),
Conv2D(128, 3, padding='same'),
BatchNormalization(scale=False, center=True),
Activation('relu'),
MaxPooling2D(2,2),
Dropout(0.2),
Conv2D(256, 3, padding='same'),
BatchNormalization(scale=False, center=True),
Activation('relu'),
Conv2D(256, 3, padding='same'),
BatchNormalization(scale=False, center=True),
Activation('relu'),
MaxPooling2D(2,2),
Dropout(0.2),
Flatten() ,
Dense(256),
BatchNormalization(scale=False, center=True),
Activation('relu'),
Dense(10, activation='softmax')
 ])
 model.summary()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 group = 'wheezy-copper-turtle-magic'
target = 'target'
cols = [c for c in train.columns if c not in ['id', target, group]]
groups = sorted(train[group].unique() )<prepare_x_and_y>
tf.keras.utils.plot_model(model, to_file="model.png", show_shapes=True )
SEED = 2019
np.random.seed(SEED)
 n_components = 6
n_clusters_per_class = n_components//2
score = []
flips = []
tr_preds = pd.Series(np.zeros(len(train)) , index = train.index)
 te_preds = pd.Series(np.zeros(len(test)) , index = test.index)
 for g in groups:
tr_idx = train[group]==g
te_idx = test[group]==g
VT = VarianceThreshold(threshold=2)
 Xtr = VT.fit_transform(train.loc[tr_idx, cols])
 Xte = VT.transform(test.loc[te_idx, cols])
 Ytr = train.loc[tr_idx, target].values
Xex = np.vstack(( Xtr, Xte))
ms, ps = MPEstimate(Xtr, Ytr, SEED)
 gm = GaussianMixture(n_components=n_components, init_params='random', covariance_type='full',
tol=0.0001, max_iter=1000, n_init=1, random_state=SEED,
means_init=ms, precisions_init=ps)
 gm.fit(Xex)
 tr_preds[tr_idx] = gm.predict_proba(Xtr)[:, n_clusters_per_class:].sum(1)
 te_preds[te_idx] = gm.predict_proba(Xte)[:, n_clusters_per_class:].sum(1)
 diff =(Ytr!=tr_preds[tr_idx].round() ).sum()
flips.append(diff)
 print('Train AUC - {}:'.format(SEED), roc_auc_score(train[target], tr_preds))<compute_test_metric>
learning_rate=0.001
batch_size = 1024
epochs = 50
steps_per_epoch = 100
validation_steps = 50
tr_preds2 = betterAucOrdering(train, tr_preds, flips, ascending=True)
 print('Train Sorted AUC 1 - {}:'.format(SEED), roc_auc_score(train[target], tr_preds2))
tr_preds2 = betterAucOrdering(train, tr_preds, flips, ascending=False)
 print('Train Sorted AUC 2 - {}:'.format(SEED), roc_auc_score(train[target], tr_preds2))
te_preds2 = betterAucOrdering(test, te_preds, flips, ascending=True )<save_to_csv>
model.compile(optimizer=Adam(lr=learning_rate),
loss=categorical_crossentropy,
metrics=['accuracy'] )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub[target] = te_preds2
sub.to_csv('submission.csv', index=False )<set_options>
def scheduler(epoch):
return learning_rate * 0.99 ** epoch
warnings.filterwarnings('ignore')
<load_from_csv>
lr_scheduler = LearningRateScheduler(scheduler)
 model_checkpoint = ModelCheckpoint('model_best_checkpoint.h5', save_best_only=True,
save_weights_only=True, monitor='val_loss', verbose=1)
 early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
 callbacks_list = [lr_scheduler, model_checkpoint, early_stopping]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<choose_model_class>
history = model.fit_generator(
 train_datagen.flow(X_train,Y_train,batch_size=batch_size),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_datagen.flow(X_valid,Y_valid),
validation_steps=validation_steps,
verbose=1,
callbacks=callbacks_list )
class MyGM(GaussianMixture):
def __init__(self, n_components=1, covariance_type='full', tol=1e-3,
reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
weights_init=None, means_init=None, precisions_init=None,
random_state=None, warm_start=False,
verbose=0, verbose_interval=10, init_clusters=None, y=None):
super().__init__(
 n_components=n_components, tol=tol, reg_covar=reg_covar,
max_iter=max_iter, n_init=n_init, init_params=init_params,
random_state=random_state, warm_start=warm_start,
verbose=verbose, verbose_interval=verbose_interval)
 self.init_clusters_ = np.asarray(init_clusters ).astype('int')
 self.y = y
def _initialize_parameters(self, X, random_state):

 n_samples, _ = X.shape
if self.init_params == 'kmeans':
resp = np.zeros(( n_samples, self.n_components))
label = cluster.KMeans(n_clusters=self.n_components, n_init=1,
random_state=random_state ).fit(X ).labels_
resp[np.arange(n_samples), label] = 1
elif self.init_params == 'random':
resp = random_state.rand(n_samples, self.n_components)
 resp /= resp.sum(axis=1)[:, np.newaxis]
elif self.init_params == 'clusters':
resp = np.zeros(( n_samples, self.n_components))
resp[np.arange(self.init_clusters_.shape[0]), self.init_clusters_] = 1
else:
raise ValueError("Unimplemented initialization method '%s'"
% self.init_params)
 self._initialize(X, resp)
 def estimate_log_ratio(self, X):
weighted_log_prob = self._estimate_weighted_log_prob(X)
 return weighted_log_prob[:, 1] - weighted_log_prob[:, 0]
def _estimate_log_prob(self, X):
base_pred = _estimate_log_gaussian_prob(
 X, self.means_, self.precisions_cholesky_, self.covariance_type)
 new_pred = base_pred
if self.y is None:
pass
else:
for i in range(base_pred.shape[0]):
yi = self.y[i]
if yi not in [0, 1]:
pass
else:
for j in range(base_pred.shape[1]):
if(j - yi)% 2 == 0:
new_pred[i, j] += np.log(0.975)
 else:
new_pred[i, j] += np.log(0.025)
 return new_pred 
<define_variables>
model.load_weights('model_best_checkpoint.h5' )
oof_preds = np.zeros(train.shape[0])
 test_preds = np.zeros(test.shape[0] )<normalization>
results = model.evaluate(X_valid, Y_valid, batch_size=batch_size, verbose=0)
 print(f"
Loss: {results[0]}")
 print(f"Accuracy: {results[1]}" )
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
train_x_0 = train3[train_index][train2['target'].values[train_index] == 0]
train_x_1 = train3[train_index][train2['target'].values[train_index] == 1]
gm0 = GaussianMixture(n_components=3)
 _ = gm0.fit(train_x_0)
 cls0 = gm0.predict(train_x_0)
 gm1 = GaussianMixture(n_components=3)
 _ = gm1.fit(train_x_1)
 cls1 = gm1.predict(train_x_1)
 train_test3 = np.concatenate([train3[train_index], test3], axis=0)
 train_cls = np.ones(train3[train_index].shape[0] ).astype(np.int)
 train_cls[train2['target'].values[train_index] == 0] = cls0 * 2
train_cls[train2['target'].values[train_index] == 1] = cls1 * 2 + 1
gm_train = MyGM(
 n_components=6, init_params='clusters', init_clusters=train_cls,
weights_init=[1/6, 1/6, 1/6, 1/6, 1/6, 1/6], reg_covar=1, tol=2, y=train2['target'].values[train_index] 
)
 _ = gm_train.fit(train3[train_index])
 gm_train.y = None
train_test_cls = gm_train.predict(train_test3)
 gm_all = MyGM(
 n_components=6, init_params='clusters', init_clusters=train_test_cls,
weights_init=[1/6, 1/6, 1/6, 1/6, 1/6, 1/6], reg_covar=1, tol=2,
y=np.concatenate([train2['target'].values[train_index], 2*np.ones(test3.shape[0] ).astype(np.int)], axis=0)
)
 gm_all.y = None
_ = gm_all.fit(train_test3)
 oof_pred6 = gm_all.predict_proba(train3[test_index])
 oof_preds[idx1[test_index]] = oof_pred6[:, 1::2].sum(axis=1)/ oof_pred6.sum(axis=1)
 test_pred6 = gm_all.predict_proba(test3)
 test_preds[idx2] += test_pred6[:, 1::2].sum(axis=1)/ test_pred6.sum(axis=1)
<compute_test_metric>
preds_dig = model.predict_classes(x_dig/255 )
roc_auc_score(train["target"], oof_preds )<save_to_csv>
print(f"Accuracy: {accuracy_score(y_dig, preds_dig)}")
 print(classification_report(y_dig, preds_dig))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = test_preds
sub.to_csv('submission.csv',index=False )<set_options>
predictions = model.predict_classes(x_test/255)
 predictions[:10]
warnings.filterwarnings('ignore')
<init_hyperparams>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
NFOLDS=5
RS=42
debug=0
params_2={'n_components' : 2,
'init_params': 'random',
'covariance_type': 'full',
'tol':0.001,
'reg_covar': 0.001,
'max_iter': 55,
'n_init': 10,
'random_state': RS,
}
params_2_qda={'n_components' : 2,
'init_params': 'random',
'covariance_type': 'full',
'tol':0.001,
'reg_covar': 0.001,
'max_iter': 50,
'n_init': 10,
'random_state': RS,
}
params_4={'n_components' : 4,
'init_params': 'random',
'covariance_type': 'full',
'tol':0.001,
'reg_covar': 0.001,
'max_iter': 55,
'n_init': 10,
'random_state': RS,
}
params_qda={'reg_param' : 0.111,
}
low=0.0067
high=1-low
rp_values = [0.8, 0.7, 0.6, 0.6]
low_vals = [0.01, 0.01, 0.01, 0.01]
high_vals = [1-low for low in low_vals]
print(low_vals, high_vals )<load_from_csv>
submission['label'] = predictions
%%time
path = Path('.. /input')
 train = pd.read_csv(path/'train.csv')
 test = pd.read_csv(path/'test.csv')
 sub = pd.read_csv(path/'sample_submission.csv' )<define_variables>
submission.to_csv("submission.csv",index=False )