Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_from_csv>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
test_csv = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 test_csv.head()<count_values>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
training_csv['target'].value_counts()<count_values>
train.isnull().sum().any()
training_csv['keyword'].value_counts()<count_missing_values>
labels = list(range(10))
training_csv['keyword'].isnull().sum()<count_values>
X = train.drop("label", axis=1 ).values
y = train["label"].values
training_csv['location'].value_counts()<count_missing_values>
from sklearn.model_selection import train_test_split
training_csv['location'].isnull().sum()<categorify>
from sklearn.model_selection import train_test_split
def clean(title):

 title = re.sub(r"\-"," ",title)
 title = re.sub(r"\+"," ",title)
 title = re.sub(r"&","and",title)
 title = re.sub(r"\|"," ",title)
 title = re.sub(r"\"," ",title)
 title = re.sub(r"\W"," ",title)
 title = title.lower()
for p in string.punctuation :
title = re.sub(r"f{p}"," ",title)
 title = re.sub(r"\s+"," ",title)
 return title<categorify>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=10 )
training_csv["cleaned_text"] = training_csv["text"].map(clean)
 training_csv.head()<load_pretrained>
X_test = test.values
tokenizer = BertTokenizer.from_pretrained('.. /input/bert-base-uncased')
<split>
from keras.utils import to_categorical
X_train,X_test,y_train,y_test = train_test_split(training_csv["cleaned_text"].values,training_csv["target"].values, random_state=0,test_size=0.1,shuffle=True )<categorify>
y_train = to_categorical(y_train)
 y_val_true = y_val.copy()
y_val = to_categorical(y_val )
class CreateDataset(Dataset):
def __init__(self, X, y, tokenizer, max_len):
self.X = X
self.y = y
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.y)
 def __getitem__(self, index):
text = self.X[index]
inputs = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=self.max_len,
truncation=True,
pad_to_max_length=True 
)
 ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.LongTensor(ids),
'mask': torch.LongTensor(mask),
'labels': torch.Tensor(self.y[index])
 } 
<create_dataframe>
X_train = X_train/255
X_val = X_val/255
X_test = X_test/255
max_len = 45
dataset_train = CreateDataset(X_train, y_train, tokenizer, max_len)
 dataset_valid = CreateDataset(X_test, y_test, tokenizer, max_len)
<load_pretrained>
X_train = X_train.reshape(-1, 28, 28, 1)
 X_val = X_val.reshape(-1, 28, 28, 1)
 X_test = X_test.reshape(-1, 28, 28, 1 )
class BERTClass(torch.nn.Module):
def __init__(self, drop_rate, otuput_size):
super().__init__()
model_config = BertConfig.from_pretrained('.. /input/bert-base-uncased', output_hidden_states=True)
 self.bert = BertModel.from_pretrained('.. /input/bert-base-uncased', config=model_config)
 self.drop = torch.nn.Dropout(drop_rate)
 self.fc = torch.nn.Linear(768, otuput_size)
 def forward(self, ids, mask):
out = self.bert(ids, attention_mask=mask)[1]
out = self.fc(self.drop(out))
return out<find_best_params>
from tensorflow.keras.preprocessing.image import ImageDataGenerator
def calculate_loss_and_accuracy(model, criterion, loader, device):

 model.eval()
loss = 0.0
total = 0
correct = 0
with torch.no_grad() :
for data in loader:
ids = data['ids'].to(device)
 mask = data['mask'].to(device)
 labels = data['labels'].to(device)
 outputs = model(ids, mask)
 loss += criterion(outputs, labels ).item()
pred = torch.round(torch.sigmoid(outputs)).cpu().numpy()
labels = labels.cpu().numpy()
total += len(labels)
 correct +=(pred == labels ).sum().item()
return loss / len(loader), correct / total
def train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, device=None):

 model.to(device)
 dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)
 dataloader_valid = DataLoader(dataset_valid, batch_size=len(dataset_valid), shuffle=False)
 log_train = []
log_valid = []
for epoch in range(num_epochs):
s_time = time.time()
model.train()
for data in dataloader_train:
ids = data['ids'].to(device)
 mask = data['mask'].to(device)
 labels = data['labels'].to(device)
 optimizer.zero_grad()
outputs = model(ids,mask)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
loss_train, acc_train = calculate_loss_and_accuracy(model, criterion, dataloader_train, device)
 loss_valid, acc_valid = calculate_loss_and_accuracy(model, criterion, dataloader_valid, device)
 log_train.append([loss_train, acc_train])
 log_valid.append([loss_valid, acc_valid])
 e_time = time.time()
print(f'epoch: {epoch + 1}, loss_train: {loss_train:.4f}, accuracy_train: {acc_train:.4f}, loss_valid: {loss_valid:.4f}, accuracy_valid: {acc_valid:.4f}, {(e_time - s_time):.4f}sec')
 return {'train': log_train, 'valid': log_valid}<choose_model_class>
image_gen = ImageDataGenerator(rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1,
horizontal_flip=False,
vertical_flip=False,
)
DROP_RATE = 0.4
OUTPUT_SIZE = 1
BATCH_SIZE = 32
NUM_EPOCHS = 2
LEARNING_RATE = 2e-5
model = BERTClass(DROP_RATE, OUTPUT_SIZE)
 criterion = torch.nn.BCEWithLogitsLoss()
optimizer = torch.optim.AdamW(params=model.parameters() , lr=LEARNING_RATE)
 device = 'cuda' if cuda.is_available() else 'cpu'
log = train_model(dataset_train, dataset_valid, BATCH_SIZE, model, criterion, optimizer, NUM_EPOCHS, device=device )<categorify>
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPool2D, Flatten, Dense, Dropout, BatchNormalization
test_csv["cleaned_text"] = test_csv["text"].map(clean)
 test_csv.head()<categorify>
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(5,5), input_shape=(28, 28, 1), padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(Conv2D(filters=32, kernel_size=(5,5), input_shape=(28, 28, 1), padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Conv2D(filters=64, kernel_size=(5,5), input_shape=(28, 28, 1), padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size=(5,5), input_shape=(28, 28, 1), padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Conv2D(filters=128, kernel_size=(5,5), input_shape=(28, 28, 1), padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(512, activation="relu"))
model.add(BatchNormalization())
 model.add(Dense(10, activation="softmax"))
model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=["accuracy"] )
class TestDataset(Dataset):
def __init__(self, X, tokenizer, max_len):
self.X = X
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.X)
 def __getitem__(self, index):
text = self.X[index]
inputs = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=self.max_len,
truncation=True,
pad_to_max_length=True 
)
 ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.LongTensor(ids),
'mask': torch.LongTensor(mask)
 } 
<create_dataframe>
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
max_len = 45
dataset_test = TestDataset(test_csv["cleaned_text"].values, tokenizer, max_len )<categorify>
check_point = ModelCheckpoint("best_model.h5", monitor="val_accuracy", verbose=1, save_best_only=True )
loader = DataLoader(dataset_test, batch_size=len(dataset_test), shuffle=False)
 model.eval()
with torch.no_grad() :
for data in loader:
ids = data['ids'].to(device)
 mask = data['mask'].to(device)
 outputs = model.forward(ids, mask)
 pred = torch.round(torch.sigmoid(outputs)).cpu().numpy()<load_from_csv>
reduce_lr = ReduceLROnPlateau(monitor="val_accuracy", patience=3, verbose=1, factor=0.5, min_lr=0.0001 )
submit_csv =pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submit_csv.head()<data_type_conversions>
submit_csv['target'] = pred.astype('int64')
 submit_csv.head(10 )<save_to_csv>
history = model.fit_generator(image_gen.flow(X_train,y_train, batch_size=64),
epochs = 50, validation_data =(X_val,y_val),
verbose = 1,
callbacks=[check_point, reduce_lr] )
submit_csv.to_csv("submission2.csv",index = False )<install_modules>
losses = pd.DataFrame(model.history.history )
!pip install transformers==3.5.1
!pip install pyspellchecker
!pip install -U joblib textblob
!python -m textblob.download_corpora|<import_modules>
print("Accuracy on validation data: {:.4f}".format(losses["val_accuracy"].max()))
import pandas as pd
import torchtext
from transformers import BertTokenizer, BertForMaskedLM, BertConfig
import transformers
import torch
from torch.utils.data import Dataset, DataLoader
from torch import optim
from torch import cuda
from sklearn.model_selection import train_test_split
import re
import string
from joblib import Parallel, delayed
from textblob import TextBlob
from textblob.translate import NotTranslated
from time import sleep<load_from_csv>
from keras.models import load_model
train_val_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<feature_engineering>
saved_model = load_model('best_model.h5' )
train_val_df = train_val_df.loc[:,["text","target"]]
test_df = test_df.loc[:,["text"]]
test_df["target"] = [0]*len(test_df["text"] )<prepare_output>
predictions = saved_model.predict_classes(X_test )
print(train_val_df)
 print(test_df.head())
 original_df = train_val_df.copy()<define_variables>
submission = pd.concat([pd.Series(range(1,28001), name ="ImageId"), submission], axis = 1 )
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not"}
def replace_typical_misspell(text):
text = text.lower()
mispellings_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
def replace(match):
return mispell_dict[match.group(0)]
return mispellings_re.sub(replace, text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : replace_typical_misspell(x))
test_df['text'] = test_df['text'].apply(lambda x : replace_typical_misspell(x))
def remove_space(string):
string = BeautifulSoup(string ).text.strip().lower()
string = re.sub(r'(( http)\S+)', 'http', string)
 string = re.sub(r'\s+', ' ', string)
 return string
train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_space(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_space(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x: remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x: remove_emoji(x))<filter>
submission.to_csv("submission.csv", index=False )
print(train_val_df.loc[31])
 print(original_df.loc[31] )<save_to_csv>
submission.to_csv("submission.csv", index=False )