Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<load_from_csv>
AUTOTUNE = tf.data.experimental.AUTOTUNE
print("Tensorflow version " + tf.__version__ )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape, test.shape<count_missing_values>
BATCH_SIZE = 64
TOTAL_DIGITS = 60000
VALIDATION_DIGITS = 8000
train.isnull().any().any()<count_values>
column_defaults = [tf.constant(0, dtype=tf.int32)for i in range(28*28+1)]
def decode_csv_line(line, has_labels):
data = tf.io.decode_csv(line, column_defaults)
 first_col = data[0]
if has_labels:
label = tf.one_hot(first_col, 10)
 else:
label = first_col
pixels = data[1:]
image = tf.reshape(pixels, [28,28])
 image = tf.cast(image, tf.float32)/ 255.0
return image, label
def augment(image, label):
std = 0.1
image = tf.expand_dims(image, axis=-1)
 batch = tf.shape(image)[0]
dy1 = tf.random.normal([batch], stddev=std)
 dx1 = tf.random.normal([batch], stddev=std)
 dy2 = tf.random.normal([batch], stddev=std)
 dx2 = tf.random.normal([batch], stddev=std)
 y1 = tf.zeros([batch])+ dy1
x1 = tf.zeros([batch])+ dx1
y2 = tf.ones([batch])+ dy2
x2 = tf.ones([batch])+ dx2
box = tf.stack([y1, x1, y2, x2], axis=-1)
 image = tf.image.crop_and_resize(image, box, tf.range(batch), [28, 28])
 return tf.squeeze(image, axis=-1), label
def blur(image, label):
image = tf.expand_dims(image, axis=-1)
 image = tf.image.resize(image, [56,56], method=tf.image.ResizeMethod.BICUBIC, antialias=True)
 image = tf.image.resize(image, [28,28], method=tf.image.ResizeMethod.BICUBIC, antialias=True)
 image = tf.squeeze(image, axis=-1)
 return(image, label)
 def load_dataset(filename, has_labels):
dataset = tf.data.TextLineDataset(filename)
 dataset = dataset.skip(1)
 dataset = dataset.map(lambda line: decode_csv_line(line, has_labels), num_parallel_calls=AUTOTUNE)
 return dataset
def get_training_dataset(filename, batch_size):
dataset = load_dataset(filename, has_labels=True ).skip(VALIDATION_DIGITS)
 dataset = dataset.cache()
dataset = dataset.shuffle(5000, reshuffle_each_iteration=True)
 dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
 dataset = dataset.map(blur)
 dataset = dataset.map(augment)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def get_validation_dataset(filename, batch_size):
dataset = load_dataset(filename, has_labels=True ).take(VALIDATION_DIGITS)
 dataset = dataset.cache()
dataset = dataset.batch(batch_size)
 dataset = dataset.map(blur)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def get_submission_dataset(filename, batch_size):
dataset = load_dataset(filename, has_labels=False)
 dataset = dataset.cache()
dataset = dataset.batch(batch_size)
 dataset = dataset.map(blur)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
train['target'].value_counts(normalize=True )<sort_values>
train_dataset = get_training_dataset("/kaggle/input/Kannada-MNIST/train.csv", batch_size=BATCH_SIZE)
 valid_dataset = get_validation_dataset("/kaggle/input/Kannada-MNIST/train.csv", batch_size=VALIDATION_DIGITS)
 submit_dataset = get_submission_dataset("/kaggle/input/Kannada-MNIST/test.csv", batch_size=1000)
 steps_per_epoch =(TOTAL_DIGITS - VALIDATION_DIGITS)// BATCH_SIZE
corrs = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
corrs = corrs[corrs['level_0'] != corrs['level_1']]
corrs.tail(30 )<normalization>
PER_ROW = 24
N = PER_ROW*4
train_digits, train_labels = dataset_to_numpy_util(train_dataset, N)
 valid_digits, valid_labels = dataset_to_numpy_util(valid_dataset, N)
 display_rows_of_digits(train_digits, train_labels, train_labels, "training digits and their labels", N//PER_ROW, PER_ROW)
 display_rows_of_digits(valid_digits, valid_labels, valid_labels, "validation digits and their labels", N//PER_ROW, PER_ROW )
X = train.drop(['ID_code', 'target'], axis=1)
 y = train['target']
X_test = test.drop(['ID_code'], axis=1)
 n_fold = 4
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42)
 repeated_folds = RepeatedStratifiedKFold(n_splits=10, n_repeats=20, random_state=42)
<split>
def make_model() :
model = tf.keras.Sequential(
 [
tf.keras.layers.Reshape(input_shape=(28,28), target_shape=(28, 28, 1), name="image"),
tf.keras.layers.SeparableConv2D(filters=12, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=16, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=20, kernel_size=6, padding='same', use_bias=False, strides=2, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=24, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=28, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=32, kernel_size=6, padding='same', use_bias=False, strides=2, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.SeparableConv2D(filters=36, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(100, use_bias=False),
tf.keras.layers.BatchNormalization(scale=False, center=True),
tf.keras.layers.Activation('relu'),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
model = make_model()
model.summary()
def lr_fn(epoch):
return LEARNING_RATE * LEARNING_RATE_EXP_DECAY**epoch
lr_decay = tf.keras.callbacks.LearningRateScheduler(lr_fn, verbose=True )
def train_model(X, X_test, y, params, folds, model_type='lgb', plot_feature_importance=False, averaging='usual', model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.loc[train_index], X.loc[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,
train_data,
num_boost_round=20000,
valid_sets = [train_data, valid_data],
verbose_eval=1000,
early_stopping_rounds = 200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_train.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_train.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict_proba(X_valid ).reshape(-1,)
 score = roc_auc_score(y_valid, y_pred_valid)
 y_pred = model.predict_proba(X_test)[:, 1]
if model_type == 'glm':
model = sm.GLM(y_train, X_train, family=sm.families.Binomial())
 model_results = model.fit()
model_results.predict(X_test)
 y_pred_valid = model_results.predict(X_valid ).reshape(-1,)
 score = roc_auc_score(y_valid, y_pred_valid)
 y_pred = model_results.predict(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, learning_rate=0.05, loss_function='Logloss', eval_metric='AUC', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred = model.predict_proba(X_test)[:, 1]
oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(roc_auc_score(y_valid, y_pred_valid))
if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores<init_hyperparams>
EPOCHS = 60
LEARNING_RATE = 0.01
LEARNING_RATE_EXP_DECAY = 0.95
display_lr_schedule(lr_fn, EPOCHS )
params = {'num_leaves': 128,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 16,
'learning_rate': 0.0123,
'boosting': 'gbdt',
'bagging_freq': 5,
'feature_fraction': 0.8201,
'bagging_seed': 11,
'reg_alpha': 1.728910519108444,
'reg_lambda': 4.9847051755586085,
'random_state': 42,
'metric': 'auc',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01077313523861969,
'min_child_weight': 19.428902804238373,
'num_threads': 4} 
<train_model>
history = model.fit(train_dataset, steps_per_epoch=steps_per_epoch, validation_data=valid_dataset, validation_steps=1, epochs=EPOCHS, callbacks=[lr_decay] )
model = lgb.LGBMClassifier(**params, n_estimators = 20000, n_jobs = -1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, stratify=y)
 model.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_valid, y_valid)], verbose=1000, early_stopping_rounds=200 )<load_pretrained>
digits, labels = dataset_to_numpy_util(valid_dataset, VALIDATION_DIGITS)
 probabilities = model.predict(digits, steps=1)
 predicted_labels = np.argmax(probabilities, axis=1)
 display_top_unrecognized(digits, predicted_labels, labels, n_rows=12, digits_per_row=24 )
eli5.show_weights(model, targets=[0, 1], feature_names=list(X_train.columns), top=40, feature_filter=lambda x: x != '<BIAS>' )<train_model>
results = None
for images, indices in submit_dataset:
predictions = model.predict(images)
 df = pd.DataFrame({'id':indices.numpy() , 'label':np.argmax(predictions, axis=-1)})
 results = results.append(df)if results is not None else df
results.to_csv("submission.csv", index=False)
 !ls -al "submission.csv"
!wc -l "submission.csv"
!head "submission.csv" -n 15
!echo "..."
!tail "submission.csv" -n 15