Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<define_variables>



 print(tf.__version__)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
dataset = get_validation_dataset()
dataset = dataset.unbatch().batch(20)
 batch = iter(dataset)
 images, labels = next(batch )<predict_on_test>
train =pd.read_csv(os.path.join(dirname,'train.csv'))
test =pd.read_csv(os.path.join(dirname,'test.csv'))
sample_submission =pd.read_csv(os.path.join(dirname,'sample_submission.csv'))
print(datetime.now())
 for j in range(start_model, end_model):
val_probabilities[j] = models[j].predict(images_ds)
 test_probabilities[j] = models[j].predict(test_images_ds)
 all_probabilities[j] = models[j].predict(images)
 print(datetime.now())
<find_best_params>
X_train=train.drop('label',axis=1)
 Y_train=train.label
X_test = test.drop('id', axis = 1)
for j in range(start_model, finished_models):
display_training_curves(historys[j].history['loss'], historys[j].history['val_loss'], 'loss', 211)
 display_training_curves(historys[j].history['sparse_categorical_accuracy'], historys[j].history['val_sparse_categorical_accuracy'], 'accuracy', 212)
 for j in range(start_model, finished_models):
print('model number:', j, ', Train Accuracy:', max(historys[j].history['sparse_categorical_accuracy']), ', Validation Accuracy:', max(historys[j].history['val_sparse_categorical_accuracy']))
for j in range(start_model, finished_models):
print('model number:', j, ', Train Loss:', min(historys[j].history['loss']), ', Validation Loss:', min(historys[j].history['val_loss'])) 
<define_variables>
X_train = X_train / 255.0
X_test = X_test / 255.0
cm_probabilities = np.zeros(( val_probabilities[0].shape))
for j in range(no_of_models):
cm_probabilities = cm_probabilities + val_probabilities[j]
cm_predictions = np.argmax(cm_probabilities, axis = -1)
 print('Correct labels: ', cm_correct_labels.shape, cm_correct_labels)
 print('Predicted labels: ', cm_predictions.shape, cm_predictions )<compute_train_metric>
Y_train = to_categorical(Y_train,num_classes=10)
 display(Y_train )
def getFitPrecisionRecall(correct_labels, predictions):
score = f1_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 precision = precision_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 recall = recall_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 return score, precision, recall 
<compute_test_metric>
X_train,X_val,y_train,y_val=train_test_split(X_train,Y_train,random_state=42,test_size=0.10)
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)))
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, cm_predictions)
 cmat =(cmat.T / cmat.sum(axis = -1)).T
display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<define_variables>
kernel_size_3 =(3,3)
 kernel_size_5 =(5,5)
 filters_32 = 32
filters_64 = 64
filters_128 = 128
filters_256 = 256
model = Sequential()
model.add(Conv2D(filters_64, kernel_size_3, activation='relu', input_shape=(28,28,1),padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(filters_64, kernel_size_3, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(filters_64, kernel_size_3, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPool2D(( 2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(filters_128, kernel_size_5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(filters_128, kernel_size_5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(filters_128, kernel_size_5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPool2D(( 2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(filters_256, kernel_size_5, activation='relu'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPool2D(( 2, 2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(256, activation='relu',kernel_regularizer=regularizers.l2(0.02)))
 model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(Dropout(0.3))
model.add(Dense(256, activation='relu',kernel_regularizer=regularizers.l2(0.02)))
 model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(Dense(512, activation='relu',kernel_regularizer=regularizers.l2(0.02)))
 model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(Dropout(0.3))
model.add(Dense(10, activation='softmax'))
def create_submission_file(filename, probabilities):
predictions = np.argmax(probabilities, axis = -1)
 print('Generating submission file...', filename)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt(filename, np.rec.fromarrays([test_ids, predictions]), fmt = ['%s', '%d'], delimiter = ',', header = 'id,label', comments = '')
<define_variables>
model.compile(loss='categorical_crossentropy',optimizer=Adadelta(learning_rate=1.0, rho=0.95),metrics=['accuracy'] )
probabilities = np.zeros(( test_probabilities[0].shape))
for j in range(no_of_models):
probabilities = probabilities + test_probabilities[j]
filename = this_run_file_prefix + 'submission.csv'
create_submission_file(filename, probabilities)
 create_submission_file('submission.csv', probabilities)
<find_best_params>
history = model.fit(X_train, y_train, batch_size = 256, epochs = 10, validation_data =(X_val, y_val), verbose = 2 )
def combine_two(correct_labels, probability_0, probability_1):
print('Start.', datetime.now())
 alphas0_to_try = np.linspace(0, 1, 101)
 best_score = -1
best_alpha0 = -1
best_alpha1 = -1
best_precision = -1
best_recall = -1
best_val_predictions = None
for alpha0 in alphas0_to_try:
alpha1 = 1.0 - alpha0
probabilities = alpha0 * probability_0 + alpha1 * probability_1
predictions = np.argmax(probabilities, axis = -1)
 score, precision, recall = getFitPrecisionRecall(correct_labels, predictions)
 if score > best_score:
best_alpha0 = alpha0
best_alpha1 = alpha1
best_score = score
best_precision = precision
best_recall = recall
best_val_predictions = predictions
return best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall<define_search_space>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range = 10,
horizontal_flip = False,
zoom_range = 0.15)
 datagen.fit(X_train )
def combine_three(correct_labels, probability_0, probability_1, probability_2):
print('Start.', datetime.now())
 alphas0_to_try = np.linspace(0, 1, 101)
 alphas1_to_try = np.linspace(0, 1, 101)
 best_score = -1
best_alpha0 = -1
best_alpha1 = -1
best_alpha2 = -1
best_precision = -1
best_recall = -1
best_val_predictions = None
for alpha0 in alphas0_to_try:
for alpha1 in alphas1_to_try:
if(alpha0 + alpha1)> 1.0:
break
alpha2 = 1.0 - alpha0 - alpha1
probabilities = alpha0 * probability_0 + alpha1 * probability_1 + alpha2 * probability_2
predictions = np.argmax(probabilities, axis = -1)
 score, precision, recall = getFitPrecisionRecall(correct_labels, predictions)
 if score > best_score:
best_alpha0 = alpha0
best_alpha1 = alpha1
best_alpha2 = alpha2
best_score = score
best_precision = precision
best_recall = recall
best_val_predictions = predictions
return best_alpha0, best_alpha1, best_alpha2, best_val_predictions, best_score, best_precision, best_recall<train_model>
BATCH_SIZE = 512
EPOCHS = 50
def get_best_combination(no_models, cm_correct_labels, val_probabilities, test_probabilities):
best_fit_score = -10000.0
best_predictions = 0
choose_filename = ''
curr_predictions = np.argmax(val_probabilities[0], axis = -1)
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, curr_predictions)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
filename = this_run_file_prefix + 'submission_0.csv'
if best_fit_score < score:
best_fit_score = score
best_predictions = curr_predictions
choose_filename = filename
create_submission_file('./submission.csv', test_probabilities[0])
 create_submission_file(filename, test_probabilities[0])
 if no_models > 1:
curr_predictions = np.argmax(val_probabilities[1], axis = -1)
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, curr_predictions)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
filename = this_run_file_prefix + 'submission_1.csv'
if best_fit_score < score:
best_fit_score = score
best_predictions = curr_predictions
choose_filename = filename
create_submission_file('./submission.csv', test_probabilities[1])
 create_submission_file(filename, test_probabilities[1])
 if no_models > 2:
curr_predictions = np.argmax(val_probabilities[2], axis = -1)
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, curr_predictions)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
filename = this_run_file_prefix + 'submission_2.csv'
if best_fit_score < score:
best_fit_score = score
best_predictions = curr_predictions
choose_filename = filename
create_submission_file('./submission.csv', test_probabilities[2])
 create_submission_file(filename, test_probabilities[2])
 if no_models > 1:
best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall = combine_two(cm_correct_labels, val_probabilities[0], val_probabilities[1])
 print('For indx', [0, 1], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[0] + best_alpha1 * test_probabilities[1]
filename = this_run_file_prefix + 'submission_01.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 if no_models > 2:
best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall = combine_two(cm_correct_labels, val_probabilities[0], val_probabilities[2])
 print('For indx', [0, 2], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[0] + best_alpha1 * test_probabilities[2]
filename = this_run_file_prefix + 'submission_02.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 best_alpha0, best_alpha1, best_val_predictions, best_score, best_precision, best_recall = combine_two(cm_correct_labels, val_probabilities[1], val_probabilities[2])
 print('For indx', [1, 2], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[1] + best_alpha1 * test_probabilities[2]
filename = this_run_file_prefix + 'submission_12.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 best_alpha0, best_alpha1, best_alpha2, best_val_predictions, best_score, best_precision, best_recall = combine_three(cm_correct_labels, val_probabilities[0], val_probabilities[1], val_probabilities[2])
 print('For indx', [0, 1, 2], 'best_alpha0:', best_alpha0, 'best_alpha1:', best_alpha1, 'best_alpha2:', best_alpha2, '.', datetime.now())
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(best_score, best_precision, best_recall))
combined_probabilities = best_alpha0 * test_probabilities[0] + best_alpha1 * test_probabilities[1] + best_alpha2 * test_probabilities[2]
filename = this_run_file_prefix + 'submission_012.csv'
if best_fit_score < best_score:
best_fit_score = best_score
best_predictions = best_val_predictions
choose_filename = filename
create_submission_file('./submission.csv', combined_probabilities)
 create_submission_file(filename, combined_probabilities)
 cmat = confusion_matrix(cm_correct_labels, best_predictions, labels = range(len(CLASSES)))
 cmat =(cmat.T / cmat.sum(axis = -1)).T
display_confusion_matrix(cmat, best_fit_score, precision, recall)
 print('Best score from all combination was', best_fit_score, '.For submission file used is', choose_filename)
 return best_predictions 
<categorify>
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=BATCH_SIZE),
epochs = EPOCHS,
shuffle=True,
validation_data =(X_val,y_val),
verbose = 1,
steps_per_epoch=X_train.shape[0] // BATCH_SIZE )
best_predictions = cm_predictions
if no_of_models > 1:
bp = get_best_combination(no_of_models, cm_correct_labels, val_probabilities, test_probabilities)
 best_predictions = bp 
<define_variables>
y_pre_test=model.predict(X_val)
 y_pre_test=np.argmax(y_pre_test,axis=1)
 y_test=np.argmax(y_val,axis=1 )
probabilities = np.zeros(( all_probabilities[0].shape))
for j in range(no_of_models):
probabilities = probabilities + all_probabilities[j]
predictions = np.argmax(probabilities, axis =-1)
 display_batch_of_images(( images, labels), predictions )<load_pretrained>
conf=confusion_matrix(y_test,y_pre_test)
 conf=pd.DataFrame(conf,index=range(0,10),columns=range(0,10))
val_probs = [cm_correct_labels, cm_predictions, test_ids, val_probabilities[0], test_probabilities[model_indx_0]]
filename = this_run_file_prefix + 'tests_vals_0.pkl'
pklfile = open(filename, 'ab')
 pickle.dump(val_probs, pklfile)
 pklfile.close()<categorify>
x=(y_pre_test-y_test!=0 ).tolist()
x=[i for i,l in enumerate(x)if l!=False]
use_correct_labels = cm_correct_labels
use_val_predictions = best_predictions<define_variables>

 results = model.predict(X_test)

results = np.argmax(results,axis = 1 )
<define_variables><EOS> 
sample_sub =pd.read_csv(os.path.join(dirname,'sample_submission.csv'))
sample_sub['label'] = results
sample_sub.to_csv('submission.csv',index=False )