Kaggle competition: Digit Recognizer
validation_data_indices = df.sample(frac=0.2 ).index
validation_df = df.loc[validation_data_indices, :].reset_index(drop=True)
 train_df = df.drop(validation_data_indices, axis=0 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 x_train, y_train = create_inputs_with_targets(list(train_df['text']), list(train_df['target']), tokenizer=bert_tokenizer, max_len=max_len)
 x_val, y_val = create_inputs_with_targets(list(validation_df['text']), list(validation_df['target']), tokenizer=bert_tokenizer, max_len=max_len)
 x_test = create_inputs_without_targets(list(test_df['text']), tokenizer=bert_tokenizer, max_len=max_len)
 print('Training dataframe size: ', len(train_df))
print('Validation dataframe size: ', len(validation_df))
print('Test dataframe size: ', len(test_df))<categorify>
%matplotlib inline
def create_model(model_name, max_len=128):
seed = 500
my_init = tf.keras.initializers.glorot_uniform(seed)
 max_len = max_len
encoder = transformers.TFAutoModel.from_pretrained(model_name)
 encoder.trainable = True
input_ids = keras.layers.Input(shape=(max_len,), dtype=tf.int32)
 attention_mask = keras.layers.Input(shape=(max_len,), dtype=tf.int32)
 sequence_output = encoder(input_ids, attention_mask=attention_mask)['last_hidden_state']
bi_lstm = tf.keras.layers.Bidirectional(keras.layers.LSTM(64, return_sequences=True))(sequence_output)
 avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
 max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
 concat = tf.keras.layers.concatenate([avg_pool, max_pool])
 dropout = tf.keras.layers.Dropout(0.3 )(concat)
 output = tf.keras.layers.Dense(1, activation="sigmoid" )(dropout)
 model = tf.keras.models.Model(
 inputs=[input_ids, attention_mask], outputs=[output] 
)
 return model<choose_model_class>
train_df = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_df = pd.read_csv(".. /input/digit-recognizer/test.csv" )
epochs = 20
lr = 2e-4
use_tpu = True
if use_tpu:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 with strategy.scope() :
model = create_model(base_model, max_len=max_len)
 optimizer = keras.optimizers.Adam(learning_rate=lr)
 model.compile(optimizer=optimizer,
loss = keras.losses.BinaryCrossentropy() ,
metrics= [keras.metrics.BinaryAccuracy() ])
 else:
model = create_model()
model.summary()<train_model>
train_df['label'].value_counts(normalize=True)
my_callbacks = [keras.callbacks.EarlyStopping(monitor='val_binary_accuracy', patience=2, mode='max', restore_best_weights=True)]
hist = model.fit(x_train,
y_train,
validation_data =(x_val, y_val),
epochs= epochs,
batch_size= 128,
callbacks = my_callbacks,
verbose= 1 )<save_to_csv>
x_train = train_df.drop(labels = ["label"],axis = 1)
 y_train = train_df['label']
del train_df
predictions = model.predict(x_test)
 ids = list(test_df['id'])
 target = [round(i[0])for i in predictions]
sub = pd.DataFrame({'id':ids, 'target':target}, index=None)
 sub.to_csv('submission.csv', index=False)
 sub<load_from_csv>
y_train = to_categorical(y_train, num_classes=10)
 y_train[:2,:]
<install_modules>
random_seed = 121
X_train, X_val, Y_train, Y_val = train_test_split(x_train, y_train, test_size = 0.15, random_state=random_seed)
!pip install transformers==3.5.1
!pip install pyspellchecker<import_modules>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(GlobalAveragePooling2D())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.2))
model.add(Dense(10, activation = "softmax"))
import pandas as pd
import torchtext
from transformers import BertTokenizer, BertForMaskedLM, BertConfig
import transformers
import torch
from torch.utils.data import Dataset, DataLoader
from torch import optim
from torch import cuda
from sklearn.model_selection import train_test_split
import re
import string 
<load_from_csv>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
train_val_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<feature_engineering>
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )
train_val_df = train_val_df.loc[:,["text","target"]]
test_df = test_df.loc[:,["text"]]
test_df["target"] = [0]*len(test_df["text"] )<prepare_output>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 50
batch_size = 86
print(train_val_df)
 print(test_df.head())
 original_df = train_val_df.copy()<define_variables>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not"}
def replace_typical_misspell(text):
text = text.lower()
mispellings_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
def replace(match):
return mispell_dict[match.group(0)]
return mispellings_re.sub(replace, text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : replace_typical_misspell(x))
test_df['text'] = test_df['text'].apply(lambda x : replace_typical_misspell(x))
def remove_space(string):
string = BeautifulSoup(string ).text.strip().lower()
string = re.sub(r'(( http)\S+)', 'http', string)
 string = re.sub(r'@\w*', '', string)
 string = re.sub(r'
string = re.sub(r'\s+', ' ', string)
 return string
train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_space(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_space(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x: remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x: remove_emoji(x)) 
<save_to_csv>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
test_df.to_csv("test.tsv", sep='\t', index=False, header=None)
 print(test_df.shape)
 train_val_df.to_csv("train_eval.tsv", sep='\t', index=False, header=None)
 print(train_val_df.shape )<categorify>
results = model.predict(test_df)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<load_pretrained><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )