Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<count_unique_values>
import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
import matplotlib.image as mpimg
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense,Conv2D,Flatten,MaxPool2D,Dropout,BatchNormalization
from keras.optimizers import Adam
from keras.callbacks import ReduceLROnPlateau
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
target_counts = np.sum(train_targets.values, axis = 1)
 print(np.unique(target_counts, return_counts = True))
target_counts[target_counts == 7] = 5
print(np.unique(target_counts, return_counts = True))<define_search_space>
IMG_SIZE = 28
N_CHANNELS = 1
N_STARTS = 3
HIDDEN_UNITS = [2048, 1024, 512]
DROPOUT_RATE = 0.5
EPOCHS = 30
BATCH_SIZE = 128
N_SPLITS = 7
res = train_targets.copy()
ss.loc[:, train_targets.columns] = 0
res.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(StratifiedKFold(n_splits = N_SPLITS,
random_state = seed,
shuffle = True ).split(train.values[:, top_feats],
target_counts)) :
print(f'Seed {seed}, Fold {n}:')
 start_time = time.time()
with strategy.scope() :
model = create_model(len(top_feats), HIDDEN_UNITS, DROPOUT_RATE)
 rlr = ReduceLROnPlateau(monitor='val_loss', factor = 0.1, patience = 3,
verbose = 0, epsilon = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'model_{seed}_{n}.hdf5', monitor = 'val_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 x_tr, x_val = train.values[tr][:, top_feats], train.values[te][:, top_feats]
y_tr, y_val = train_targets.astype(float ).values[tr], train_targets.astype(float ).values[te]
history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = EPOCHS,
batch_size = BATCH_SIZE, callbacks = [rlr, ckp], verbose = 0)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'model_{seed}_{n}.hdf5')
 ss.loc[:, train_targets.columns] += model.predict(test_features.values[:, top_feats])
 res.loc[te, train_targets.columns] += model.predict(train.values[te][:, top_feats])
 print('Best Validation Loss:\t', hist['val_loss'].min())
 print('Cost Time:\t', time.time() - start_time)
 print('-' * 50)
 K.clear_session()
del model, history, hist
gc.collect()
ss.loc[:, train_targets.columns] /=(( n + 1)* N_STARTS)
 res.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
train_df = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test_df = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')
 pred_df = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv' )
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(train_targets.loc[:, _target], res.loc[:, _target]))
print(f'OOF Metric: {np.mean(metrics)}' )<compute_test_metric>
train_df = train_df.append(test_df )
metrics = []
res.loc[train['cp_type'] == 1, train_targets.columns] = 0
for _target in train_targets.columns:
metrics.append(log_loss(train_targets.loc[:, _target], res.loc[:, _target]))
print(f'OOF Metric with postprocessing: {np.mean(metrics)}' )<feature_engineering>
print(f'Training set: {train_df.shape}')
 print(f'To be Predicted: {pred_df.shape}' )
ss.loc[test['cp_type'] == 1, train_targets.columns] = 0<save_to_csv>
X_train = train_df.drop(['label'], axis = 1)
 Y_train = train_df['label']
X_pred = pred_df.drop(['id'], axis = 1 )
ss.to_csv('submission.csv', index = False )<import_modules>
X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size=0.05 )
import numpy as np
import pandas as pd
import os
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<load_from_csv>
X_train, X_test, X_pred = X_train.apply(lambda x: x/255), X_test.apply(lambda x: x/255), X_pred.apply(lambda x: x/255 )
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = train_target_df.columns[1:]
N_TARGETS = len(target_cols )<define_variables>
Y_train, Y_test = pd.get_dummies(Y_train), pd.get_dummies(Y_test )
SEED = 2020
EPOCHS = 30
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 5
LR = 0.00045
N_TARGETS = len(target_cols )<define_variables>
X_train = X_train.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)
 X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS )
def seed_everything(seed):
np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<compute_test_metric>
print(f'Training images: {X_train.shape}')
 print(f'Testing images: {X_test.shape}' )
def multi_log_loss(y_true, y_pred):
losses = []
for col in y_true.columns:
losses.append(log_loss(y_true.loc[:, col], y_pred.loc[:, col]))
return np.mean(losses )<data_type_conversions>
Y_train = Y_train.to_numpy()
def preprocess_df(data):
data['cp_type'] =(data['cp_type'] == 'trt_cp' ).astype(int)
 data['cp_dose'] =(data['cp_dose'] == 'D2' ).astype(int)
 return data<prepare_x_and_y>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=8,
zoom_range = 0.15,
width_shift_range=0.15,
height_shift_range=0.15,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
x_train = preprocess_df(train_df.drop(columns="sig_id"))
x_test =preprocess_df(test_df.drop(columns="sig_id"))
y_train = train_target_df.drop(columns="sig_id")
 N_FEATURES = x_train.shape[1]<choose_model_class>
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu', input_shape=(28, 28, 1)))
 model.add(Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu'))
model.add(BatchNormalization(momentum=0.15))
model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='SAME', activation='relu'))
model.add(Dropout(rate=0.3))
model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu'))
model.add(BatchNormalization(momentum=0.15))
model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu'))
model.add(Dropout(rate=0.3))
model.add(Flatten())
 model.add(Dense(128, activation = "relu"))
model.add(Dropout(0.40))
model.add(Dense(64, activation = "relu"))
model.add(Dropout(0.40))
model.add(Dense(10, activation = "softmax"))
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(N_TARGETS, activation="sigmoid"))
])
 model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = LR), loss='binary_crossentropy', metrics=["accuracy"])
 return model<train_model>
model.compile(optimizer="adam", loss=['categorical_crossentropy'], metrics=['accuracy'] )
def build_train(resume_models = None, repeat_number = 0, folds = 7, skip_folds = 0):
models = []
oof_preds = y_train.copy()
kfold = KFold(folds, shuffle = True)
 for fold,(train_ind, val_ind)in enumerate(kfold.split(x_train)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'binary_crossentropy', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'auto')
 checkpoint_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 model = create_model()
model.fit(x_train.values[train_ind],
y_train.values[train_ind],
validation_data=(x_train.values[val_ind], y_train.values[val_ind]),
callbacks = [cb_lr_schedule, cb_checkpt],
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=2 
)
 model.load_weights(checkpoint_path)
 oof_preds.loc[val_ind, :] = model.predict(x_train.values[val_ind])
 models.append(model)
 return models, oof_preds<train_model>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
models = []
oof_preds = []
seed_everything(SEED)
 for i in range(REPEATS):
m, oof = build_train(repeat_number = i, folds=FOLDS)
 models = models + m
oof_preds.append(oof)
<compute_test_metric>
batch_size=32
epochs = 25
mean_oof_preds = y_train.copy()
mean_oof_preds.loc[:, target_cols] = 0
for i, p in enumerate(oof_preds):
print(f"Repeat {i + 1} OOF Log Loss: {multi_log_loss(y_train, p)}")
 mean_oof_preds.loc[:, target_cols] += p[target_cols]
mean_oof_preds.loc[:, target_cols] /= len(oof_preds)
 print(f"Mean OOF Log Loss: {multi_log_loss(y_train, mean_oof_preds)}")
 mean_oof_preds.loc[x_train['cp_type'] == 0, target_cols] = 0
print(f"Mean OOF Log Loss(ctl adjusted): {multi_log_loss(y_train, mean_oof_preds)}" )<save_to_csv>
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_test,Y_test),
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[learning_rate_reduction])
test_preds = sub.copy()
test_preds[target_cols] = 0
for model in models:
test_preds.loc[:,target_cols] += model.predict(x_test)
 test_preds.loc[:,target_cols] /= len(models)
 test_preds.loc[x_test['cp_type'] == 0, target_cols] = 0
test_preds.to_csv('submission.csv', index=False )<import_modules>
cfm = confusion_matrix(np.argmax(Y_test.to_numpy() , axis=1), np.argmax(model.predict(X_test), axis=1))
cfm = pd.DataFrame(cfm,index=range(0,10),columns=range(0,10))
cfm
import numpy as np
import pandas as pd
import os 
<install_modules>
preds = model.predict(X_pred.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS))
! pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl<install_modules>
pred_df['label'] = np.argmax(preds, axis=1 )
!pip install.. /input/iterative-stratification/iterative_stratification-0.1.6-py3-none-any.whl 
<feature_engineering>
preds = pred_df[['id', 'label']]
<set_options><EOS> 
preds.to_csv('sub.csv', index=False )