Kaggle competition: Digit Recognizer
!pip install tensorflow_text 
<define_variables>
%matplotlib inline
bert_model_name = 'bert_en_uncased_L-12_H-768_A-12'
map_name_to_handle = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_base/2',
'electra_small':
'https://tfhub.dev/google/electra_small/2',
'electra_base':
'https://tfhub.dev/google/electra_base/2',
'experts_pubmed':
'https://tfhub.dev/google/experts/bert/pubmed/2',
'experts_wiki_books':
'https://tfhub.dev/google/experts/bert/wiki_books/2',
'talking-heads_base':
'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',
}
map_model_to_preprocess = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_preprocess/3',
'electra_small':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'electra_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'experts_pubmed':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'experts_wiki_books':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'talking-heads_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
}
tfhub_handle_encoder = map_name_to_handle[bert_model_name]
tfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]
print(f'BERT model selected : {tfhub_handle_encoder}')
 print(f'Preprocess model auto-selected: {tfhub_handle_preprocess}' )<categorify>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import seaborn as sns
bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)
 text_test = ['this is such an amazing movie!']
text_preprocessed = bert_preprocess_model(text_test)
 print(f'Keys : {list(text_preprocessed.keys())}')
 print(f'Shape : {text_preprocessed["input_word_ids"].shape}')
 print(f'Word Ids : {text_preprocessed["input_word_ids"][0, :12]}')
 print(f'Input Mask : {text_preprocessed["input_mask"][0, :12]}')
 print(f'Type Ids : {text_preprocessed["input_type_ids"][0, :12]}' )<train_model>
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.optimizers import RMSprop, Adam
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools
bert_model = hub.KerasLayer(tfhub_handle_encoder)
 bert_results = bert_model(text_preprocessed)
 print(f'Loaded BERT: {tfhub_handle_encoder}')
 print(f'Pooled Outputs Shape:{bert_results["pooled_output"].shape}')
 print(f'Pooled Outputs Values:{bert_results["pooled_output"][0, :12]}')
 print(f'Sequence Outputs Shape:{bert_results["sequence_output"].shape}')
 print(f'Sequence Outputs Values:{bert_results["sequence_output"][0, :12]}' )<categorify>
PATH = '.. /input/'
def build_classifier_model() :
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string, name='text')
 preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation='sigmoid', name='classifier' )(net)
 return tf.keras.Model(text_input, net)
 classifier_model = build_classifier_model()
classifier_model.compile(optimizer=tf.keras.optimizers.Adam(3e-5),
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=tf.metrics.BinaryAccuracy())
 checkpoint=tf.keras.callbacks.ModelCheckpoint('model.h5',monitor='val_loss',save_best_only=True)
 es=tf.keras.callbacks.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)
 history = classifier_model.fit(train.text,train.target,validation_split=0.2,epochs=10,callbacks=[checkpoint,es],batch_size=8 )<load_pretrained>
df_train = pd.read_csv(f'{PATH}train.csv')
 df_test = pd.read_csv(f'{PATH}test.csv' )
classifier_model.load_weights('./model.h5')
 pred=classifier_model.predict(test.text )<count_values>
X_train = df_train.drop('label', axis=1 ).values
X_test = df_test.values
Y_train = df_train.label
print(f'Number of training Examples, {X_train.shape[0]}')
 print(f'Number of test Examples, {X_test.shape[0]}')
 print(f'Number of classes, {np.unique(Y_train)}' )
pd.DataFrame(np.where(pred>0.5,1,0)).value_counts()<save_to_csv>
X_train = X_train.reshape(-1, 28, 28, 1)
 X_test = X_test.reshape(-1, 28, 28, 1 )
pd.DataFrame({
'id':test.id,
'target':np.where(pred>0.5,1,0)[:,0]
} ).to_csv('submission.csv',index=False )<load_from_csv>
print('Integer Valued Labels')
 print(Y_train[:10])
 Y_train = np_utils.to_categorical(Y_train, 10)
 print('One Hot Labels')
 print(Y_train[:10] )
train_data = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 train_data.head(5 )<load_from_csv>
X_train = X_train.astype('float32')/ 255
X_test = X_test.astype('float32')/ 255
test_data = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 test_data.head(5 )<install_modules>
X_train, X_valid = X_train[7000:], X_train[:7000]
Y_train, Y_valid = Y_train[7000:], Y_train[:7000]
!pip install BeautifulSoup4<string_transform>
model = Sequential()
model.add(Flatten(input_shape=X_train.shape[1:]))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
stop = set(stopwords.words('english'))
stop.update(list(string.punctuation))
def clean_tweets(text):
re1 = re.compile(r' +')
 x1 = text.lower().replace('
'nbsp;', ' ' ).replace('
', "
" ).replace('quot;', "'" ).replace(
 '<br />', "
" ).replace('\"', '"' ).replace('<unk>', 'u_n' ).replace(' @.@ ', '.' ).replace(
 ' @-@ ', '-' ).replace('\', ' \\ ')
 text = re1.sub(' ', html.unescape(x1))
text = unicodedata.normalize('NFKD', text ).encode('ascii', 'ignore' ).decode('utf-8', 'ignore')
 soup = BeautifulSoup(text, 'html.parser')
 text = soup.get_text()
text = re.sub('\[[^]]*\]', '', text)
 text = re.sub(r'http\S+', '', text)
 text = text.replace("@", "")
 text = text.replace("
text = re.sub(r'[^a-zA-Z ]', '', text)
 final_text = []
for word in text.split() :
if word.strip().lower() not in stop:
final_text.append(word.strip().lower())
 text = " ".join(final_text)
 lemmatizer = WordNetLemmatizer()
text = " ".join([lemmatizer.lemmatize(word)for word in text.split() ])
 text = " ".join([lemmatizer.lemmatize(word, pos = 'v')for word in text.split() ])
 text = re.sub("\d", "num", text)
 return text.lower()
train_data['prep_text'] = train_data['text'].apply(clean_tweets)
 train_data['prep_text'].head(5 )<feature_engineering>
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )
test_data['text'] = test_data['text'].apply(clean_tweets)
 test_data['text'].head(5 )<train_model>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5',
verbose=1, save_best_only=True)
 hist = model.fit(X_train, Y_train, batch_size=128, epochs=10,
validation_split=0.2, callbacks=[checkpointer],
verbose=1, shuffle=True )
vocab_size = 1000
tokenizer = Tokenizer(num_words = vocab_size, oov_token = 'UNK')
 tokenizer.fit_on_texts(list(train_data['prep_text'])+ list(test_data['text']))<data_type_conversions>
model.load_weights('mnist.model.best.hdf5' )
X_train_ohe = tokenizer.texts_to_matrix(train_data['prep_text'], mode = 'binary')
 X_test_ohe = tokenizer.texts_to_matrix(test_data['text'], mode = 'binary')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_ohe.shape}")
 print(f"X_test shape: {X_test_ohe.shape}")
 print(f"y_train shape: {y_train.shape}" )<split>
score = model.evaluate(X_valid, Y_valid, verbose=0)
 accuracy = 100 * score[1]
print(f'Test Accuracy {accuracy:.4f}' )
X_train_ohe, X_val_ohe, y_train, y_val = train_test_split(X_train_ohe, y_train, random_state = 42, test_size = 0.2)
 print(f"X_train shape: {X_train_ohe.shape}")
 print(f"X_val shape: {X_val_ohe.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )<choose_model_class>
def submit_result(model, X_test):
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("mnist_datagen.csv",index=False)
def setup_model() :
model = Sequential()
model.add(layers.Dense(1, activation='sigmoid', input_shape=(vocab_size,)))
 model.compile(optimizer=optimizers.RMSprop(lr=0.001),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
 return model
model = setup_model()
model.summary()<train_model>
cnn_model = Sequential()
cnn_model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same',
activation='relu', input_shape=X_train.shape[1:]))
cnn_model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same',
activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same',
activation='relu'))
cnn_model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same',
activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Flatten())
 cnn_model.add(Dense(512, activation='relu'))
cnn_model.add(Dropout(0.2))
cnn_model.add(Dense(512, activation='relu'))
cnn_model.add(Dropout(0.2))
cnn_model.add(Dense(10, activation='softmax'))
cnn_model.summary()
history = model.fit(X_train_ohe, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_ohe, y_val))<compute_test_metric>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 cnn_model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'] )
_, accuracy = model.evaluate(X_val_ohe, y_val )<data_type_conversions>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.cnn.hdf5',
verbose=1, save_best_only=True)
 hist = cnn_model.fit(X_train, Y_train, batch_size=128, epochs=10,
validation_split=0.2, callbacks=[checkpointer],
verbose=1, shuffle=True )
X_train_wc = tokenizer.texts_to_matrix(train_data['prep_text'], mode = 'count')
 X_test_wc = tokenizer.texts_to_matrix(test_data['text'], mode = 'count')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_wc.shape}")
 print(f"X_test shape: {X_test_wc.shape}")
 print(f"y_train shape: {y_train.shape}")
<split>
cnn_model.load_weights('mnist.model.best.cnn.hdf5' )
X_train_wc, X_val_wc, y_train, y_val = train_test_split(X_train_wc, y_train, random_state = 42, test_size = 0.2)
 print(f"X_train shape: {X_train_wc.shape}")
 print(f"X_val shape: {X_val_wc.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )<train_model>
score = cnn_model.evaluate(X_valid, Y_valid, verbose=0)
 accuracy = 100 * score[1]
print(f'Test Accuracy {accuracy:.4f}' )
history = model.fit(X_train_wc, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_wc, y_val))<compute_test_metric>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.2)
 print(f'Shape of the training data {X_train.shape}')
 print(f'Shape of the validation dat {X_val.shape}' )
_, accuracy = model.evaluate(X_val_wc, y_val )<data_type_conversions>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False 
)
 datagen.fit(X_train )
X_train_freq = tokenizer.texts_to_matrix(train_data['prep_text'], mode = 'freq')
 X_test_freq = tokenizer.texts_to_matrix(test_data['text'], mode = 'freq')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_freq.shape}")
 print(f"X_test shape: {X_test_freq.shape}")
 print(f"y_train shape: {y_train.shape}" )<split>
batch_size=128
checkpointer = ModelCheckpoint(filepath='mnist.model.best.cnn.aug.hdf5',
verbose=1, save_best_only=True)
 hist = cnn_model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
epochs=10, validation_data=(X_val, Y_val), callbacks=[checkpointer],
verbose=2, shuffle=True, steps_per_epoch=X_train.shape[0] // batch_size )
X_train_freq, X_val_freq, y_train, y_val = train_test_split(X_train_freq, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_freq.shape}")
 print(f"X_val shape: {X_val_freq.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )<train_model>
cnn_model.load_weights('mnist.model.best.cnn.aug.hdf5' )
history = model.fit(X_train_freq, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_freq, y_val))<train_model>
score = cnn_model.evaluate(X_valid, Y_valid, verbose=0)
 accuracy = 100 * score[1]
print(f'Test Accuracy {accuracy:.4f}' )
vectorizer = TfidfVectorizer(max_features = vocab_size)
 vectorizer.fit(list(train_data['prep_text'])+ list(test_data['text']))
X_train_tfidf = vectorizer.transform(list(train_data['prep_text'])).toarray()
X_test_tfidf = vectorizer.transform(list(test_data['text'])).toarray()
y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape {X_train_tfidf.shape}")
 print(f"X_test shape {X_test_tfidf.shape}")
 print(f"y_train shape {y_train.shape}" )<split>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
X_train_tfidf, X_val_tfidf, y_train, y_val = train_test_split(X_train_tfidf, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_tfidf.shape}")
 print(f"X_val shape: {X_val_tfidf.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )<train_model>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.cnn.aug.ann.hdf5',
verbose=1, save_best_only=True)
 hist = cnn_model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
epochs=20, validation_data=(X_val, Y_val), callbacks=[checkpointer, learning_rate_reduction],
verbose=2, shuffle=True, steps_per_epoch=X_train.shape[0] // batch_size)
history = model.fit(X_train_tfidf, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_tfidf, y_val))<define_variables>
cnn_model.load_weights('mnist.model.best.cnn.aug.ann.hdf5' )
embedding_dict={}
with open('.. /input/glovetwitter27b100dtxt/glove.twitter.27B.100d.txt','r')as f:
for line in f:
values=line.split()
word = values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()<categorify>
score = cnn_model.evaluate(X_valid, Y_valid, verbose=0)
 accuracy = 100 * score[1]
print(f'Test Accuracy {accuracy:.4f}' )
vocab_size = 10000
tokenizer = Tokenizer(num_words = vocab_size, oov_token = 'UNK')
 tokenizer.fit_on_texts(list(train_data['prep_text'])+ list(test_data['text']))
max_len = 15
X_train_seq = tokenizer.texts_to_sequences(train_data['prep_text'])
 X_test_seq = tokenizer.texts_to_sequences(test_data['text'])
 X_train_seq = pad_sequences(X_train_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 X_test_seq = pad_sequences(X_test_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_seq.shape}")
 print(f"X_test shape: {X_test_seq.shape}")
 print(f"y_train shape: {y_train.shape}" )<split>
submit_result(cnn_model, X_test )
X_train_seq, X_val_seq, y_train, y_val = train_test_split(X_train_seq, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_seq.shape}")
 print(f"X_val shape: {X_val_seq.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )<count_unique_values>
cnn_bn_model = Sequential()
cnn_bn_model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 cnn_bn_model.add(BatchNormalization())
 cnn_bn_model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu'))
cnn_bn_model.add(BatchNormalization())
 cnn_bn_model.add(MaxPooling2D(pool_size=(2,2)))
 cnn_bn_model.add(Dropout(0.25))
cnn_bn_model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
cnn_bn_model.add(BatchNormalization())
 cnn_bn_model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
cnn_bn_model.add(BatchNormalization())
 cnn_bn_model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 cnn_bn_model.add(Dropout(0.25))
cnn_bn_model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
cnn_bn_model.add(BatchNormalization())
 cnn_bn_model.add(Dropout(0.25))
cnn_bn_model.add(Flatten())
 cnn_bn_model.add(Dense(256, activation = "relu"))
cnn_bn_model.add(BatchNormalization())
 cnn_bn_model.add(Dropout(0.25))
cnn_bn_model.add(Dense(10, activation = "softmax"))
cnn_bn_model.summary()
num_words = len(tokenizer.word_index)
 print(f"Number of unique words: {num_words}" )<categorify>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
embedding_matrix=np.zeros(( num_words,100))
for word,i in tokenizer.word_index.items() :
if i < num_words:
emb_vec = embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i] = emb_vec<choose_model_class>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 cnn_bn_model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'] )
n_latent_factors = 100
model_glove = Sequential()
model_glove.add(layers.Embedding(num_words, n_latent_factors, weights = [embedding_matrix],
input_length = max_len, trainable=True))
model_glove.add(layers.Flatten())
 model_glove.add(layers.Dropout(0.5))
model_glove.add(layers.Dense(1, activation='sigmoid'))
model_glove.summary()<train_model>
checkpointer = ModelCheckpoint(filepath='final.model.hdf5',
verbose=1, save_best_only=True)
 hist = cnn_bn_model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
epochs=20, validation_data=(X_val, Y_val), callbacks=[checkpointer, learning_rate_reduction],
verbose=2, shuffle=True, steps_per_epoch=X_train.shape[0] // batch_size)
model_glove.compile(optimizer = optimizers.RMSprop(lr=0.001),
loss = losses.binary_crossentropy,
metrics = [metrics.binary_accuracy])
 history = model_glove.fit(X_train_seq,
y_train,
epochs=20,
batch_size=512,
validation_data=(X_val_seq, y_val))<categorify>
cnn_bn_model.load_weights('final.model.hdf5' )
max_len = 15
X_train_seq = tokenizer.texts_to_sequences(train_data['prep_text'])
 X_test_seq = tokenizer.texts_to_sequences(test_data['text'])
 X_train_seq = pad_sequences(X_train_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 X_test_seq = pad_sequences(X_test_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_seq.shape}")
 print(f"X_test shape: {X_test_seq.shape}")
 print(f"y_train shape: {y_train.shape}
")
 n_latent_factors = 100
model_glove = Sequential()
model_glove.add(layers.Embedding(num_words, n_latent_factors, weights = [embedding_matrix],
input_length = max_len, trainable=True))
model_glove.add(layers.Flatten())
 model_glove.add(layers.Dropout(0.5))
model_glove.add(layers.Dense(1, activation='sigmoid'))
print(f"{model_glove.summary() }
")
 model_glove.compile(optimizer = optimizers.RMSprop(lr=0.001),
loss = losses.binary_crossentropy,
metrics = [metrics.binary_accuracy])
 history = model_glove.fit(X_train_seq,
y_train,
epochs=20,
batch_size=512 )<split>
score = cnn_bn_model.evaluate(X_valid, Y_valid, verbose=0)
 accuracy = 100 * score[1]
print(f'Test Accuracy {accuracy:.4f}' )
<save_to_csv><EOS> 
submit_result(cnn_bn_model, X_test )