Kaggle competition: Digit Recognizer
def extract_keywords(text):
potential_keywords = []
TOP_KEYWORD = -1
pos_tag = ['ADJ', 'NOUN', 'PROPN']
doc = nlp_spacy(text)
 for i in doc:
if i.pos_ in pos_tag:
potential_keywords.append(i.text)
 document_embed = sentence_enc([text])
 potential_embed = sentence_enc(potential_keywords)
 vector_distances = cosine_similarity(document_embed, potential_embed)
 keyword = [potential_keywords[i] for i in vector_distances.argsort() [0][TOP_KEYWORD:]]
return keyword
def keyword_filler(keyword, text):
if pd.isnull(keyword):
try:
keyword = extract_keywords(text)[0]
except:
keyword = ''
return keyword<data_type_conversions>
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep' )
df_train.keyword = pd.DataFrame(list(map(keyword_filler, df_train.keyword, df_train.text)) ).astype(str)
 df_test.keyword = pd.DataFrame(list(map(keyword_filler, df_test.keyword, df_test.text)) ).astype(str)
 print('Null Training Keywords => ', df_train['keyword'].isnull().any())
 print('Null Test Keywords => ', df_test['keyword'].isnull().any() )<split>
df_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 df_test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
X_train, X_val, y_train, y_val = train_test_split(df_train[['text','keyword']],
df_train.target,
test_size=0.2,
random_state=42)
 X_train.shape, X_val.shape<create_dataframe>
X_train = np.array(df_train.drop(['label'], axis=1), dtype="float32")/ 255.0
X_train = X_train.reshape(-1, 28, 28, 1)
 Y_train = to_categorical(df_train['label'], num_classes = 10)
 X_test = np.array(df_test, dtype="float32")/ 255.0
X_test = X_test.reshape(-1, 28, 28, 1 )
train_ds = tf.data.Dataset.from_tensor_slices(( dict(X_train), y_train))
val_ds = tf.data.Dataset.from_tensor_slices(( dict(X_val), y_val))
test_ds = tf.data.Dataset.from_tensor_slices(dict(df_test[['text','keyword']]))<categorify>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.05, random_state=34 )
AUTOTUNE = tf.data.experimental.AUTOTUNE
BUFFER_SIZE = 1000
BATCH_SIZE = 32
RANDOM_SEED = 319
def configure_dataset(dataset, shuffle=False, test=False):
if shuffle:
dataset = dataset.cache() \ 
.shuffle(BUFFER_SIZE, seed=RANDOM_SEED, reshuffle_each_iteration=True)\ 
.batch(BATCH_SIZE, drop_remainder=True)\ 
.prefetch(AUTOTUNE)
 elif test:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=False)\ 
.prefetch(AUTOTUNE)
 else:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=True)\ 
.prefetch(AUTOTUNE)
 return dataset<create_dataframe>
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(10, activation = "softmax"))
a3 = configure_dataset(train_ds, shuffle=True)
 dict3 = []
for elem in a3:
dict3.append(elem[0]['text'][0])
 dict3[:10]<prepare_x_and_y>
datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
train_ds = configure_dataset(train_ds, shuffle=True)
 val_ds = configure_dataset(val_ds)
 test_ds = configure_dataset(test_ds, test=True )<train_model>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
del X_train, X_val, y_train, y_val, df_train, df_test, train_full, test_full<define_variables>
epochs=50
batch_size=128
history = model.fit(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs,
validation_data =(X_val,Y_val),
verbose = 2,
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[learning_rate_reduction] )
bert_encoder_path = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4"
bert_preprocessor_path = "https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3"
keyword_embedding_path = "https://tfhub.dev/google/nnlm-en-dim128-with-normalization/2"<load_pretrained>
Y_pred = model.predict(X_val)
 Y_pred_classes = np.argmax(Y_pred,axis = 1)
 Y_true = np.argmax(Y_val,axis = 1 )
bert_encoder = hub.KerasLayer(bert_encoder_path, trainable=True, name="BERT_Encoder")
 bert_preprocessor = hub.KerasLayer(bert_preprocessor_path, name="BERT_Preprocessor")
 nnlm_embed = hub.KerasLayer(keyword_embedding_path, name="NNLM_Embedding" )<categorify>
errors =(Y_pred_classes - Y_true != 0)
 Y_pred_classes_errors = Y_pred_classes[errors]
Y_pred_errors = Y_pred[errors]
Y_true_errors = Y_true[errors]
X_val_errors = X_val[errors]
kernel_initializer = tf.keras.initializers.GlorotNormal(seed=319)
 def create_model() :
text_input = Input(shape=() , dtype=tf.string, name="text")
 encoder_inputs = bert_preprocessor(text_input)
 encoder_outputs = bert_encoder(encoder_inputs)
 pooled_output = encoder_outputs["pooled_output"]
bert_branch = Dropout(0.1,
seed=319,
name="BERT_Dropout" )(pooled_output)
 keyword_input = Input(shape=() , dtype=tf.string, name='keyword')
 keyword_embed = nnlm_embed(keyword_input)
 keyword_flat = Flatten(name="Keyword_Flatten" )(keyword_embed)
 keyword_dense1 = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Keyword_Dense1"
)(keyword_flat)
 keyword_branch1 = Dropout(0.5,
seed=319,
name='Keyword_dropout1'
)(keyword_dense1)
 keyword_dense2 = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Keyword_Dense2"
)(keyword_branch1)
 keyword_branch2 = Dropout(0.5,
seed=319,
name='Keyword_dropout2'
)(keyword_dense2)
 keyword_dense3 = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Keyword_Dense3"
)(keyword_branch2)
 keyword_branch3 = Dropout(0.5,
seed=319,
name='Keyword_dropout3'
)(keyword_dense3)
 merge = concatenate([bert_branch, keyword_branch3], name="Concatenate")
 dense = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Merged_Dense" )(merge)
 dropout = Dropout(0.5,
seed=319,
name="Merged_Dropout"
)(dense)
 clf = Dense(1,
activation="sigmoid",
kernel_initializer=kernel_initializer,
name="Classifier"
)(dropout)
 return Model([text_input, keyword_input],
clf,
name="BERT_Classifier" )<init_hyperparams>
Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)
 true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))
delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors
sorted_dela_errors = np.argsort(delta_pred_true_errors)
 most_important_errors = sorted_dela_errors[-6:]
display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors )
EPOCHS = 3
LEARNING_RATE = 5e-5
STEPS_PER_EPOCH = int(train_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 VAL_STEPS = int(val_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 TRAIN_STEPS = STEPS_PER_EPOCH * EPOCHS
WARMUP_STEPS = int(TRAIN_STEPS * 0.1)
 adamw_optimizer = create_optimizer(
 init_lr=LEARNING_RATE,
num_train_steps=TRAIN_STEPS,
num_warmup_steps=WARMUP_STEPS,
optimizer_type='adamw'
)<train_model>
results = model.predict(X_test)
 results = np.argmax(results, axis = 1)
 results = pd.Series(results, name="Label" )
<save_to_csv><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"), results],axis = 1)
 submission.to_csv("submission.csv",index=False )