Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_test_metric>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.models import Sequential, load_model
from keras.layers import Conv2D, Dense, Flatten, MaxPool2D, Dropout
from keras.optimizers import Adam
from keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import RandomizedSearchCV
submission(bert_classifier, test_ds )<load_from_url>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 print(train.head())
 test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 print(test.head() )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py 
<define_variables>
X_train = train.drop(labels = ['label'], axis = 1)
 y_train = train.label
del train
SEED = 1002
def seed_everything(seed):
np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )<load_from_csv>
X_train = X_train / 255.0
test = test / 255.0
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 for i, val in enumerate(train.iloc[:2]["text"].to_list()):
print("Tweet {}: {}".format(i+1, val))<categorify>
y_train = to_categorical(y_train, num_classes = 10 )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train,
test_size = 0.1,
random_state = 10,
stratify = y_train )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(learning_rate=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose = 0)
 nets = 3
model = [0] * nets
for i in range(nets):
model[i] = Sequential()
model[i].add(Conv2D(filters = 32, kernel_size = 5, padding = 'same',
activation = 'relu',
input_shape =(28, 28, 1)))
 model[i].add(MaxPool2D())
 if i > 0:
model[i].add(Conv2D(filters = 48, kernel_size = 5, padding = 'same',
activation = 'relu'))
model[i].add(MaxPool2D())
 if i > 1:
model[i].add(Conv2D(filters = 64, kernel_size = 5, padding = 'same',
activation = 'relu'))
model[i].add(MaxPool2D(padding = 'same'))
model[i].add(Flatten())
 model[i].add(Dense(256, activation = 'relu'))
model[i].add(Dense(10, activation = 'softmax'))
model[i].compile(optimizer = 'adam', loss = 'categorical_crossentropy',
metrics = ['accuracy'])
 history = [0] * nets
names = ['CNN-1', 'CNN-2', 'CNN-3']
epochs = 20
for i in range(nets):
history[i] = model[i].fit(X_train, y_train, batch_size = 100,
epochs = epochs,
validation_data =(X_val, y_val),
callbacks = [annealer], verbose = 0)
 print('{0}: Epochs = {1: d}, Train accuracy = {2:.5f}, Validation accuracy = {3:.5f}'.format(names[i],
epochs, max(history[i].history['accuracy']),
max(history[i].history['val_accuracy'])))
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<categorify>
nets = 7
model = [0] * nets
for i in range(nets):
model[i] = Sequential()
model[i].add(Conv2D(i*8+8, kernel_size = 5,
padding = 'same',
activation = 'relu',
input_shape =(28, 28, 1)))
 model[i].add(MaxPool2D())
 model[i].add(Conv2D(i*16+16, kernel_size = 5,
padding = 'same',
activation = 'relu'))
model[i].add(MaxPool2D())
 model[i].add(Flatten())
 model[i].add(Dense(256, activation = 'relu'))
model[i].add(Dense(10, activation = 'softmax'))
model[i].compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
 history = [0] * nets
names = []
for i in range(nets):
names.append(i*8+8)
 for i in range(nets):
history[i] = model[i].fit(X_train, y_train,
batch_size = 100,
epochs = epochs,
validation_data =(X_val, y_val),
callbacks = [annealer], verbose = 0)
 print('CNN {0: d} maps: Epochs = {1: d}, Train accuracy = {2:.5f}, Validation accuracy = {3:.5f}'.format(names[i],
epochs, max(history[i].history['accuracy']), max(history[i].history['val_accuracy'])) )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
nets = 8
model = [0] * nets
for i in range(nets):
model[i] = Sequential()
model[i].add(Conv2D(48, kernel_size = 5,
padding = 'same',
activation = 'relu',
input_shape =(28, 28, 1)))
 model[i].add(MaxPool2D())
 model[i].add(Conv2D(96, kernel_size = 5,
activation = 'relu'))
model[i].add(MaxPool2D())
 model[i].add(Flatten())
 if i>0:
model[i].add(Dense(2**(i+4), activation = 'relu'))
model[i].add(Dense(10, activation = 'softmax'))
model[i].compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
 history = [0] * nets
names = [0]
for i in range(nets - 1):
names.append(2**(i+5))
for i in range(nets):
history[i] = model[i].fit(X_train, y_train,
batch_size = 100,
epochs = epochs,
validation_data =(X_val, y_val),
callbacks = [annealer], verbose = 0)
 print('CNN {0: d}N: Epochs {1: d}, Training accuracy {2:.5f}, Validation accuracy {3:.5f}'.format(names[i],
epochs, max(history[i].history['accuracy']), max(history[i].history['val_accuracy'])) )
checkpoint = ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.1,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)<save_to_csv>
nets = 8
model = [0] * nets
names = []
for i, n in enumerate(range(8)) :
names.append(f'{n*10}%')
 for i in range(nets):
model[i] = Sequential()
model[i].add(Conv2D(48, kernel_size = 5,
padding = 'same',
activation = 'relu',
input_shape =(28, 28, 1)))
 model[i].add(MaxPool2D())
 model[i].add(Dropout(i*0.1))
model[i].add(Conv2D(96, kernel_size = 5,
activation = 'relu'))
model[i].add(MaxPool2D())
 model[i].add(Dropout(i*0.1))
model[i].add(Flatten())
 model[i].add(Dense(256, activation = 'relu'))
model[i].add(Dropout(i*0.1))
model[i].add(Dense(10, activation = 'sigmoid'))
model[i].compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
 history = [0] * nets
for i in range(nets):
history[i] = model[i].fit(X_train, y_train,
batch_size = 100,
epochs = epochs,
validation_data =(X_val, y_val),
callbacks = [annealer],
verbose = 0)
 print('CNN Dropouts = {0}: Epochs = {1: d}, Training accuracy = {2:.5f}, Validation accuracy = {3:.5f}'.format(names[i],
epochs, max(history[i].history['accuracy']), max(history[i].history['val_accuracy'])) )
test_pred = model.predict(test_input)
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
def create_model(optimizer = 'adam', activation = 'relu'):
model = Sequential()
model.add(Conv2D(48, kernel_size = 5,
padding = 'same',
activation = activation,
input_shape =(28, 28, 1)))
 model.add(MaxPool2D())
 model.add(Dropout(0.4))
model.add(Conv2D(96, kernel_size = 5,
activation = activation))
model.add(MaxPool2D())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(256, activation = activation))
model.add(Dropout(0.4))
model.add(Dense(10, activation = 'sigmoid'))
model.compile(optimizer = optimizer,
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
 return model
model = KerasClassifier(build_fn = create_model)
 params = dict(optimizer = ['sgd', 'adam'], activation = ['relu', 'tanh'],
batch_size = [50, 100, 150, 200], epochs = [10, 20, 30, 50])
 random_search = RandomizedSearchCV(model, param_distributions = params, cv = 3,
n_iter = 10)
 random_search_results = random_search.fit(X_train, y_train,
validation_data =(X_val, y_val))
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
print('Best: {0: 5f} using {1}'.format(random_search_results.best_score_,
random_search_results.best_params_))
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
early_stopping = EarlyStopping(monitor = 'val_loss', patience = 5,
verbose = 0,
restore_best_weights = True)
 annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose = 0)
 model_checkpoint = ModelCheckpoint('Digit_Recognizer.hdf5', monitor='val_accuracy',
verbose=1, save_best_only=True,
mode='max')
 cnn = create_model(optimizer = 'adam', activation = 'relu')
 results = cnn.fit(X_train, y_train, validation_data =(X_val, y_val),
batch_size = 50, epochs = 30,
callbacks = [early_stopping, model_checkpoint, annealer] )
<feature_engineering><EOS> 
model = load_model('Digit_Recognizer.hdf5')
 model.evaluate(X_val, y_val)
 pred = model.predict(test)
 pred = np.argmax(pred, axis=1)
 pred = pd.Series(pred,name="Label")
 result = pd.concat([pd.Series(range(1,28001),name = "ImageId"),pred],axis = 1)
 result.to_csv("./digit_recognizer.csv",index=False )