Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_test_metric>
!pip install emnist
test_loss, test_acc = CNNmodel.evaluate(X_test_cnn, y_test_cnn, verbose=2)
 print('test loss:',test_loss)
 print('test acc:',test_acc )<prepare_x_and_y>
import matplotlib.pyplot as plt,seaborn as sns,pandas as pd,numpy as np
from keras.models import Sequential, load_model
from keras.layers.core import Dense, Dropout, Activation
from keras.layers import Conv2D, MaxPooling2D,MaxPool2D,Flatten,BatchNormalization
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from emnist import extract_training_samples
from emnist import extract_test_samples
from keras.optimizers import Adam
embedding_matrix = np.zeros(( len(word_index)+ 1, embedding_dim))
for word, i in word_index.items() :
if word in word2vec_model:
embedding_matrix[i] = np.asarray(word2vec_model.wv[word] )<choose_model_class>
x_train, y_train = extract_training_samples('digits')
 x_test, y_test = extract_test_samples('digits' )
embedding_layer = Embedding(len(word_index)+1,
embedding_dim,
weights = [embedding_matrix],
input_length = max_sequence_length,
trainable = False )<choose_model_class>
in_train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 ex_y_train = in_train_data["label"]
ex_x_train = in_train_data.drop(labels = ["label"],axis = 1 )
model = Sequential()
model.add(embedding_layer)
 model.add(Conv1D(filters=150, kernel_size=3, strides=1, padding='valid', activation = 'relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Flatten())
 model.add(Dense(embedding_dim, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(cnn_label.shape[1], activation='sigmoid'))
model.summary()<train_model>
X_train = x_train.reshape(240000, 28, 28,1)
 X_test = x_test.reshape(40000, 28, 28,1)
 ex_x_train = ex_x_train.values.reshape(42000,28,28,1)
 X_train = np.vstack(( X_train, ex_x_train))
print(X_train.shape )
model.compile(optimizer='adam', loss=losses.binary_crossentropy, metrics=['accuracy'])
 history = model.fit(X_cnn, y_cnn, epochs=10, validation_data=(X_val_cnn, y_val_cnn))<compute_test_metric>
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32' )
test_loss, test_acc = model.evaluate(X_test_cnn, y_test_cnn, verbose=2)
 print('test loss:',test_loss)
 print('test acc:',test_acc )<import_modules>
X_train /= 255
X_test /= 255
tf.__version__<import_modules>
y_train = np.concatenate([y_train,ex_y_train.values])
 print(y_train.shape )
hub.__version__<load_from_url>
n_classes = 10
print("Shape before one-hot encoding: ", y_train.shape)
 Y_train = np_utils.to_categorical(y_train, n_classes)
 Y_test = np_utils.to_categorical(y_test, n_classes)
 print("Shape after one-hot encoding: ", Y_train.shape )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(MaxPool2D(pool_size = 2,strides=2))
model.add(Conv2D(filters=48, kernel_size=(5,5), padding='valid', activation='relu'))
model.add(MaxPool2D(pool_size = 2,strides=2))
model.add(Flatten())
 model.add(Dense(120, activation='relu'))
model.add(Dense(84, activation='relu'))
model.add(Dense(10, activation='softmax'))
adam = Adam(lr=5e-4)
 model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam )
import tensorflow as tf
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization<categorify>
reduce_lr = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.2,
min_lr=1e-6 )
def bert_encode(texts, bert_layer, max_len=128):
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len - 2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 input_ids = tokens + [0]* pad_len
all_tokens.append(input_ids)
 masks = [1]*len(input_sequence)+ [0]* pad_len
all_masks.append(masks)
 segments = [0]* max_len
all_segments.append(segments)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len = 128, lr = 1e-5):
input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 dense_out = Dense(1,activation="relu" )(pooled_output)
 drop_out = tf.keras.layers.Dropout(0.8 )(dense_out)
 out = Dense(1,activation="sigmoid" )(pooled_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 adam = tf.keras.optimizers.Adam(lr)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
def plot_curve(history):
plt.plot(history.history['accuracy'], label='accuracy')
 plt.plot(history.history['val_accuracy'], label='val_accuracy')
 plt.xlabel('Epoch')
 plt.ylabel('Accuracy')
 plt.ylim([0.5,1])
 plt.legend()
plt.show()<choose_model_class>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1)
 history = datagen.fit(X_train )
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<load_from_csv>
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=100), steps_per_epoch=len(X_train)/100,
epochs=20, validation_data=(X_test, Y_test), callbacks=[reduce_lr] )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 train_input = bert_encode(train.text.values, bert_layer, max_len=128)
 train_labels = np.array(train.target )<load_from_csv>
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 test_input = bert_encode(test.text.values, bert_layer, max_len=128)
 model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
score = model.evaluate(X_test, Y_test, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = np.round(test_pred ).astype('int')
 submission.to_csv('submission.csv', index=False)
 submission.groupby('target' ).count()<load_from_csv>
test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 data.sample(10 )<count_duplicates>
test_data = test_data.values
test_data = test_data.reshape(28000, 28, 28,1)
 test_data = test_data.astype('float32')
 test_data /= 255
print("Test data matrix shape", test_data.shape )
text = data.text
duplicates = data[text.isin(text[text.duplicated() ])].sort_values(by='text')
 conflicting_check = pd.DataFrame(duplicates.groupby(['text'] ).target.mean())
 conflicting_check.sample(10 )<filter>
y_pred = model.predict_classes(test_data, verbose=0)
 print(y_pred )
conflicting = conflicting_check.loc[(conflicting_check.target != 1)&(conflicting_check.target != 0)].index
data = data.drop(data[text.isin(conflicting)].index)
 print('Conflicting samples count:', conflicting.shape[0] )<set_options>
i = 9713
predicted_value = np.argmax(model.predict(X_test[i].reshape(1,28, 28,1)))
 print('predicted value:',predicted_value)
 plt.imshow(X_test[i].reshape([28, 28]), cmap='Greys_r' )
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )<install_modules>
submissions=pd.DataFrame({"ImageId": list(range(1,len(y_pred)+1)) ,
"Label": y_pred})
 submissions.to_csv("LeNet_CNN.csv", index=False )
!pip install transformers<define_variables>
!pip install emnist
sentences = data.text.values
labels =data.target.values<load_pretrained>
import matplotlib.pyplot as plt,seaborn as sns,pandas as pd,numpy as np
from keras.models import Sequential, load_model
from keras.layers.core import Dense, Dropout, Activation
from keras.layers import Conv2D, MaxPooling2D,MaxPool2D,Flatten,BatchNormalization
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from emnist import extract_training_samples
from emnist import extract_test_samples
from keras.optimizers import Adam
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )<categorify>
x_train, y_train = extract_training_samples('digits')
 x_test, y_test = extract_test_samples('digits' )
print(' Original: ', sentences[0])
 print('Tokenized: ', tokenizer.tokenize(sentences[0]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])) )<define_variables>
in_train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 ex_y_train = in_train_data["label"]
ex_x_train = in_train_data.drop(labels = ["label"],axis = 1 )
max_len = 0
for sent in sentences:
input_ids = tokenizer.encode(sent, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max tweet length: ', max_len )<categorify>
X_train = x_train.reshape(240000, 28, 28,1)
 X_test = x_test.reshape(40000, 28, 28,1)
 ex_x_train = ex_x_train.values.reshape(42000,28,28,1)
 X_train = np.vstack(( X_train, ex_x_train))
print(X_train.shape )
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original: ', sentences[0])
 print('Token IDs:', input_ids[0] )<split>
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32' )
SPLIT = 0.999
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(SPLIT * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))<load_pretrained>
X_train /= 255
X_test /= 255
batch_size = 32
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)<load_pretrained>
y_train = np.concatenate([y_train,ex_y_train.values])
 print(y_train.shape )
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()<choose_model_class>
n_classes = 10
print("Shape before one-hot encoding: ", y_train.shape)
 Y_train = np_utils.to_categorical(y_train, n_classes)
 Y_test = np_utils.to_categorical(y_test, n_classes)
 print("Shape after one-hot encoding: ", Y_train.shape )
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8
)<init_hyperparams>
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(MaxPool2D(pool_size = 2,strides=2))
model.add(Conv2D(filters=48, kernel_size=(5,5), padding='valid', activation='relu'))
model.add(MaxPool2D(pool_size = 2,strides=2))
model.add(Flatten())
 model.add(Dense(120, activation='relu'))
model.add(Dense(84, activation='relu'))
model.add(Dense(10, activation='softmax'))
adam = Adam(lr=5e-4)
 model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam )
epochs = 2
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )<compute_test_metric>
reduce_lr = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.2,
min_lr=1e-6 )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<define_variables>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1)
 history = datagen.fit(X_train )
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
logits = outputs[1]
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
output = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = output[0]
logits = output[1]
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )<create_dataframe>
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=100), steps_per_epoch=len(X_train)/100,
epochs=20, validation_data=(X_test, Y_test), callbacks=[reduce_lr] )
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 df_stats<load_from_csv>
test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print('Number of test sentences: {:,}
'.format(test_data.shape[0]))
sentences = test_data.text.values
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 batch_size = 32
prediction_data = TensorDataset(input_ids, attention_masks,)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )<predict_on_test>
score = model.evaluate(X_test, Y_test, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 print(' DONE.' )<define_variables>
test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
flat_predictions = np.concatenate(predictions, axis=0)
 flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()<save_to_csv>
test_data = test_data.values
test_data = test_data.reshape(28000, 28, 28,1)
 test_data = test_data.astype('float32')
 test_data /= 255
print("Test data matrix shape", test_data.shape )
submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission.target = flat_predictions
submission.to_csv('submission.csv', index=False )<set_options>
y_pred = model.predict_classes(test_data, verbose=0)
 print(y_pred )
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 warnings.filterwarnings("ignore")
 eng_stopwords = set(stopwords.words("english"))<load_from_csv>
i = 9713
predicted_value = np.argmax(model.predict(X_test[i].reshape(1,28, 28,1)))
 print('predicted value:',predicted_value)
 plt.imshow(X_test[i].reshape([28, 28]), cmap='Greys_r' )
train_df = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test_df = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 print("Training Shape rows = {}, columns = {}".format(train_df.shape[0],train_df.shape[1]))
print("Testing Shape rows = {}, columns = {}".format(test_df.shape[0],test_df.shape[1]))<count_missing_values>
submissions=pd.DataFrame({"ImageId": list(range(1,len(y_pred)+1)) ,
"Label": y_pred})
 submissions.to_csv("LeNet_CNN.csv", index=False )