Kaggle competition: Digit Recognizer
def restore_grouping_same_words_without_embeddings(tweet):
tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 return tweet<define_variables>
%matplotlib inline
np.random.seed(0 )
def remove_urls(tweet):
tweet = re.sub("https?:\/\/t.co\/[A-Za-z0-9]*", '', tweet)
 return tweet<drop_column>
train = pd.read_csv(r'/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv(r'/kaggle/input/digit-recognizer/test.csv' )
def remove_emojis(tweet):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 tweet = emoji_pattern.sub(r'', tweet)
 return tweet<categorify>
Y = train['label']
X = train.drop('label',axis=1 )
def remove_punctuations(tweet):
tweet = tweet.translate(str.maketrans('', '', string.punctuation))
return tweet<drop_column>
X = X / 255.0
test = test / 255.0
%%time
def clean(tweet):
tweet = clean_special_characters(tweet)
 tweet = restore_contractions(tweet)
 tweet = restore_character_entity_references(tweet)
 tweet = restore_typos_slang_and_informal_abbreviations(tweet)
 tweet = restore_hashtags_usernames(tweet)
 tweet = restore_acronyms(tweet)
 tweet = restore_grouping_same_words_without_embeddings(tweet)
 tweet = remove_urls(tweet)
 tweet = remove_emojis(tweet)
 tweet = remove_punctuations(tweet)
 tweet = re.sub("\s+", ' ', tweet)
 return tweet
train_df['text_cleaned'] = train_df['text'].apply(lambda s : clean(s))
test_df['text_cleaned'] = test_df['text'].apply(lambda s : clean(s))<concatenate>
Y = to_categorical(Y,10 )
concat_df = pd.concat([train_df, test_df], axis = 0 ).reset_index(drop = True)
 MAX_SEQ_LEN = len(max(concat_df.text_cleaned, key = len))
print('The maximum length of each sequence is:', MAX_SEQ_LEN )<define_variables>
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.2 )
AUTOTUNE = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 32
SEED = 42<count_unique_values>
model = keras.models.Sequential([
keras.layers.Conv2D(32,(5,5),input_shape=(28,28,1),activation='relu',padding='same'),
keras.layers.BatchNormalization(axis=1),
keras.layers.MaxPooling2D(2,2),
keras.layers.Conv2D(32,(5,5),activation='relu',padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.Conv2D(64,(3,3),activation='relu',padding='same'),
keras.layers.BatchNormalization(axis=1),
keras.layers.Conv2D(64,(3,3),activation='relu',padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.Flatten() ,
keras.layers.Dense(256,activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
keras.layers.Dense(10,activation='softmax')
 ] )
K = 2
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
 DISASTER = train_df['target_relabeled'] == 1
print('Whole Training Set Shape = {}'.format(train_df.shape[0]))
print('Whole Training Set Unique keyword Count = {}'.format(train_df['keyword'].nunique()))
print('Whole Training Set Target Rate(Disaster/Not Disaster)= {}'.format(train_df[DISASTER]['target_relabeled'].count() /train_df[~DISASTER]['target_relabeled'].count()))
train_ds_list = []
valid_ds_list = []
for fold,(trn_idx, val_idx)in enumerate(skf.split(train_df['text_cleaned'], train_df['target_relabeled'])) :
print('
Fold {} Training Set Shape = {} - Validation Set Shape = {}'.format(fold, train_df.loc[trn_idx, 'text_cleaned'].shape[0], train_df.loc[val_idx, 'text_cleaned'].shape[0]))
print('Fold {} Training Set Unique keyword Count = {} - Validation Set Unique keyword Count = {}'.format(fold, train_df.loc[trn_idx, 'keyword'].nunique() , train_df.loc[val_idx, 'keyword'].nunique()))
print('Fold {} Training Set Target Rate(Disaster/Not Disaster)= {}'.format(fold, train_df.loc[trn_idx, 'target_relabeled'][DISASTER].count() /train_df.loc[trn_idx,'target_relabeled'][~DISASTER].count()))
trn_ds = tf.data.Dataset.from_tensor_slices(( train_df.loc[trn_idx, 'text_cleaned'].values, train_df.loc[trn_idx, 'target_relabeled'].values))
train_ds_list.append(trn_ds)
 val_ds = tf.data.Dataset.from_tensor_slices(( train_df.loc[val_idx, 'text_cleaned'].values, train_df.loc[val_idx, 'target_relabeled'].values))
valid_ds_list.append(val_ds )<split>
model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'] )
test_ds_raw = tf.data.Dataset.from_tensor_slices(test_df['text_cleaned'].values)
 for text in test_ds_raw.take(5):
print(f'Review: {text}' )<train_model>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
BERT_MODEL = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3'
PREPROCESS_MODEL = 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2'
print(f'BERT model selected : {BERT_MODEL}')
 print(f'Preprocess model auto-selected: {PREPROCESS_MODEL}' )<choose_model_class>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
def make_bert_preprocess_model(sentence_features, seq_length=128):

 input_segments = [
tf.keras.layers.Input(shape=() , dtype=tf.string, name=ft)
 for ft in sentence_features]
bert_preprocess = hub.load(PREPROCESS_MODEL)
 tokenizer = hub.KerasLayer(bert_preprocess.tokenize, name='tokenizer')
 segments = [tokenizer(s)for s in input_segments]
truncated_segments = segments
packer = hub.KerasLayer(bert_preprocess.bert_pack_inputs,
arguments=dict(seq_length=seq_length),
name='packer')
 model_inputs = packer(truncated_segments)
 return tf.keras.Model(input_segments, model_inputs )<categorify>
epochs = 250
batch_size=64
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_test,Y_test),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
,callbacks=[learning_rate_reduction] )
def build_classifier_model() :
inputs = dict(
 input_word_ids=tf.keras.layers.Input(shape=(None,), dtype=tf.int32, name='word_ids'),
input_mask=tf.keras.layers.Input(shape=(None,), dtype=tf.int32, name='mask'),
input_type_ids=tf.keras.layers.Input(shape=(None,), dtype=tf.int32, name='type_ids'), 
)
 encoder = hub.KerasLayer(BERT_MODEL, trainable=True, name='BERT_encoder')
 outputs = encoder(inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation=None, name='classifier' )(net)
 return tf.keras.Model(inputs, net, name='none_em_proc_model' )<categorify>
predictions = model.predict(test)
 predictions = np.argmax(predictions,axis = 1)
 predictions = pd.Series(predictions,name="Label" )
def build_classifier_model_em_proc() :
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string, name='text')
 preprocessing_layer = hub.KerasLayer(PREPROCESS_MODEL, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(BERT_MODEL, trainable=True, name='BERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation=None, name='classifier' )(net)
 return tf.keras.Model(text_input, net, name='em_proc_model' )<define_variables>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),predictions],axis = 1)
 submission.to_csv("submissions_mnist.csv",index=False)
 print("Your file is saved." )
train_ds = train_ds_list[0]
train_ds = train_ds.shuffle(SEED ).batch(BATCH_SIZE)
 train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
 valid_ds = valid_ds_list[0]
valid_ds = valid_ds.batch(BATCH_SIZE)
 valid_ds = valid_ds.cache().prefetch(buffer_size=AUTOTUNE )<choose_model_class>
%matplotlib inline
np.random.seed(0 )
loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
 metrics = tf.metrics.BinaryAccuracy()<choose_model_class>
train = pd.read_csv(r'/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv(r'/kaggle/input/digit-recognizer/test.csv' )
EPOCHS = 10
INIT_LR = 3e-5
steps_per_epoch = tf.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * EPOCHS
num_warmup_steps = int(0.1 * num_train_steps)
 optimizer = optimization.create_optimizer(init_lr = INIT_LR,
num_train_steps = num_train_steps,
num_warmup_steps = num_warmup_steps,
optimizer_type = 'adamw' )<choose_model_class>
Y = train['label']
X = train.drop('label',axis=1 )
model_em_proc.compile(optimizer=optimizer,
loss=loss,
metrics=metrics )<train_model>
X = X / 255.0
test = test / 255.0
print('Training model with embedded preprocess model')
 history_em_proc = model_em_proc.fit(x=train_ds,
validation_data=valid_ds,
epochs = EPOCHS )<predict_on_test>
Y = to_categorical(Y,10 )
test_ds = test_ds_raw.batch(BATCH_SIZE ).cache().prefetch(buffer_size=AUTOTUNE)
 predict_result_em_proc = tf.sigmoid(model_em_proc.predict(test_ds))
print(predict_result_em_proc )<categorify>
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.2 )
train_ds = train_ds_list[1]
train_ds = train_ds.shuffle(SEED ).batch(BATCH_SIZE)
 train_ds = train_ds.map(lambda x, y:(preprocess_model(x), y))
train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
 valid_ds = valid_ds_list[1]
valid_ds = valid_ds.batch(BATCH_SIZE)
 valid_ds = valid_ds.map(lambda x, y:(preprocess_model(x), y))
valid_ds = valid_ds.cache().prefetch(buffer_size=AUTOTUNE )<choose_model_class>
model = keras.models.Sequential([
keras.layers.Conv2D(32,(5,5),input_shape=(28,28,1),activation='relu',padding='same'),
keras.layers.BatchNormalization(axis=1),
keras.layers.MaxPooling2D(2,2),
keras.layers.Conv2D(32,(5,5),activation='relu',padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.Conv2D(64,(3,3),activation='relu',padding='same'),
keras.layers.BatchNormalization(axis=1),
keras.layers.Conv2D(64,(3,3),activation='relu',padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.Flatten() ,
keras.layers.Dense(256,activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
keras.layers.Dense(10,activation='softmax')
 ] )
steps_per_epoch = tf.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * EPOCHS
num_warmup_steps = int(0.1 * num_train_steps)
 optimizer = optimization.create_optimizer(init_lr = INIT_LR,
num_train_steps = num_train_steps,
num_warmup_steps = num_warmup_steps,
optimizer_type = 'adamw' )<choose_model_class>
model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'] )
model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics )<train_model>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
print('Training model without embedded preprocess model')
 history_model = model.fit(x=train_ds,
validation_data=valid_ds,
epochs = EPOCHS )<predict_on_test>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
test_ds = test_ds_raw.batch(BATCH_SIZE ).cache().prefetch(buffer_size=AUTOTUNE)
 test_ds = test_ds.map(lambda x: preprocess_model(x))
predict_result = tf.sigmoid(model.predict(test_ds))
print(predict_result )<save_to_csv>
epochs = 250
batch_size=64
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_test,Y_test),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
,callbacks=[learning_rate_reduction] )
result_1 = predict_result_em_proc.numpy()
result_2 = predict_result.numpy()
result = []
for i in range(len(result_1)) :
result.append(( result_1[i] + result_2[i])/2)
 result = np.round(result)
 sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 ids = sample_submission.id
final_submission = pd.DataFrame(np.c_[ids, result.astype('int')], columns = ['id', 'target'])
 final_submission.to_csv('submission.csv', index = False)
 final_submission.describe()<set_options>
predictions = model.predict(test)
 predictions = np.argmax(predictions,axis = 1)
 predictions = pd.Series(predictions,name="Label" )
<load_from_csv><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),predictions],axis = 1)
 submission.to_csv("submissions_mnist.csv",index=False)
 print("Your file is saved." )