Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPool2D, BatchNormalization
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
warnings.simplefilter(action='ignore', category=FutureWarning)
<set_options>
train_path = '.. /input/digit-recognizer/train.csv'
test_path = '.. /input/digit-recognizer/test.csv'
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
train_data = pd.read_csv(train_path)
 test_data = pd.read_csv(test_path )
train_dir = ".. /input/nlp-getting-started/train.csv"
df = pd.read_csv(train_dir)
 x =(df['text'] 
.str.lower() 
.str.replace('\x89Ûª|Ûª', "'")
.str.replace('
|\x89.|\x9d *', ' ')
.str.replace('&gt;', ">")
.str.replace('&lt;', "<")
.str.replace('&amp;', " and ")
.str.replace('won't', 'will not')
.str.replace('can't', 'cannot')
.str.replace('i'm', 'i am')
.str.replace('ain't', 'is not')
.str.replace('hwy.', 'highway')
.str.replace('(\w+)'ll', '\g<1> will')
.str.replace('(\w+)n't', '\g<1> not')
.str.replace('(\w+)'ve', '\g<1> have')
.str.replace('(\w+)'s', '\g<1> is')
.str.replace('(\w+)'re', '\g<1> are')
.str.replace('(\w+)'d', '\g<1> would')
.str.replace('(\w+)'m', '\g<1> am')
.str.replace('<3', 'love')
.str.replace('w/e', 'whatever')
.str.replace('w/', 'with')
.str.replace('\b', ' ')
.str.replace('-', ' ')
.str.replace(' *', ' ')
.str.strip() 
.to_numpy() 
)
 y = df['target'].to_numpy()
x_train, x_val, y_train, y_val = train_test_split(x, y, train_size = 0.7, random_state = 1000 )<load_pretrained>
img_rows, img_cols = 28, 28
num_classes = train_data.label.nunique()
tokenizer = AutoTokenizer.from_pretrained('vinai/bertweet-base',
normalization=True,
use_fast = False,
add_special_tokens=True,
pad_to_max_length=True,
return_attention_mask=True)
 train_token = tokenizer(x_train.tolist() ,
padding="max_length",
truncation=True,
return_tensors = 'tf' ).data
val_token = tokenizer(x_val.tolist() ,
padding="max_length",
truncation=True,
return_tensors = 'tf' ).data
train_features = {x: train_token[x] for x in tokenizer.model_input_names}
train_tf_dataset = tf.data.Dataset.from_tensor_slices(( train_features, y_train))
train_tf_dataset = train_tf_dataset.shuffle(len(x_train)).batch(32 ).prefetch(tf.data.AUTOTUNE)
 val_features = {x: val_token[x] for x in tokenizer.model_input_names}
val_tf_dataset = tf.data.Dataset.from_tensor_slices(( val_features, y_val))
val_tf_dataset = val_tf_dataset.shuffle(len(x_val)).batch(32 ).prefetch(tf.data.AUTOTUNE )<choose_model_class>
def data_prep(raw_data, img_row, img_col, num_classes):
out_y = to_categorical(raw_data.label, num_classes)
 num_images = raw_data.shape[0]
x_as_array = raw_data.values[:, 1:]
x_as_matrix = x_as_array.reshape(num_images, img_row, img_col, 1)
 out_x = x_as_matrix/255.0
return out_x, out_y
with strategy.scope() :
bert_model = TFRobertaModel.from_pretrained("vinai/bertweet-base")
 def build_model(hidden_n, drop = 0.3, lr = 1e-5, weight_decay = 1e-6):
with strategy.scope() :
input_ids = tf.keras.Input(shape=(128,),dtype='int32', name = 'input_ids')
 attention_masks = tf.keras.Input(shape=(128,),dtype='int32', name = 'attention_mask')
 output = bert_model([input_ids,attention_masks])[1]
output = tf.keras.layers.Dense(hidden_n,activation='relu' )(output)
 output = tf.keras.layers.Dropout(drop )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks], outputs = output)
 f1_score = tfa.metrics.F1Score(1, threshold = 0.5)
 model.compile(tfa.optimizers.AdamW(weight_decay = weight_decay, lr=lr),
loss='binary_crossentropy',
metrics=['accuracy', f1_score])
 model.layers[2].trainable = False
return model<init_hyperparams>
X, y = data_prep(train_data, img_rows, img_cols, num_classes )
grid = [{'hidden_n': 16, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 1e-6},
{'hidden_n': 32, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 1e-6},
{'hidden_n': 32, 'drop': 0.3, 'lr': 5e-6, 'weight_decay': 1e-6},
{'hidden_n': 32, 'drop': 0.25, 'lr': 1e-5, 'weight_decay': 5e-6},
{'hidden_n': 32, 'drop': 0.35, 'lr': 1e-5, 'weight_decay': 5e-7},
{'hidden_n': 64, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 1e-6},
{'hidden_n': 64, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 5e-6},
]
def run_loop(**kwargs):
model = build_model(**kwargs)
 history = model.fit(train_tf_dataset,
validation_data = val_tf_dataset,
epochs = 2)
 return {**kwargs, 'model': model, 'f1': history.history['val_f1_score'][-1][0]}
results = [run_loop(**params)for params in grid]<load_from_csv>
model = Sequential()
TEST_PATH = ".. /input/nlp-getting-started/test.csv"
df = pd.read_csv(TEST_PATH)
 x =(df['text'] 
.str.lower() 
.str.replace('\x89Ûª|Ûª', "'")
.str.replace('
|\x89.|\x9d *', ' ')
.str.replace('&gt;', ">")
.str.replace('&lt;', "<")
.str.replace('&amp;', " and ")
.str.replace('won't', 'will not')
.str.replace('can't', 'cannot')
.str.replace('i'm', 'i am')
.str.replace('ain't', 'is not')
.str.replace('hwy.', 'highway')
.str.replace('(\w+)'ll', '\g<1> will')
.str.replace('(\w+)n't', '\g<1> not')
.str.replace('(\w+)'ve', '\g<1> have')
.str.replace('(\w+)'s', '\g<1> is')
.str.replace('(\w+)'re', '\g<1> are')
.str.replace('(\w+)'d', '\g<1> would')
.str.replace('(\w+)'m', '\g<1> am')
.str.replace('<3', 'love')
.str.replace('w/e', 'whatever')
.str.replace('w/', 'with')
.str.replace('\b', ' ')
.str.replace('-', ' ')
.str.replace(' *', ' ')
.str.strip() 
.to_numpy() 
)
 token = tokenizer(x.tolist() ,
padding="max_length",
truncation=True,
return_tensors = 'tf' ).data
df['p'] = model.predict(token)
 df['target'] =(df['p'] > 0.5 ).astype(int )<save_to_csv>
model.add(Conv2D(filters=40, kernel_size=(3,3), padding='Same', activation='relu', input_shape=(img_rows, img_cols, 1)))
 model.add(MaxPool2D(2,2))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters=60, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization(axis=1))
model.add(Dropout(0.25))
model.add(Conv2D(filters=80, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(BatchNormalization(axis=1))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(BatchNormalization(axis=1))
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization(axis=1))
model.add(Dropout(0.50))
model.add(BatchNormalization(axis=1))
model.add(Dense(10, activation='softmax'))
model.summary()
df[['id', 'target']].to_csv("submissions.csv", index = False )<install_modules>
model.compile(loss=categorical_crossentropy,
optimizer=Adam(0.001),
metrics=['accuracy'] )
<set_options>
%matplotlib inline<load_from_csv>
history = model.fit(X, y, epochs=20, validation_split=0.2 )
sample_submission = pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip')
 test_labels = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip')
 train = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/train.csv.zip')
 test = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/test.csv.zip' )<import_modules>
test_num_img = test_data.shape[0]
test_data_array = test_data.values
test_data_matrix = test_data_array.reshape(test_num_img, 28,28,1)
 test_data_matrix = test_data_matrix/255
from sklearn.model_selection import train_test_split<import_modules>
preds = model.predict_classes(test_data_matrix )
from sklearn.model_selection import train_test_split<split>
output = pd.DataFrame({'ImageId': list(range(1, len(preds)+1)) ,
'Label':preds})
 output.to_csv('output2.csv', index=False )
X = train.comment_text.values
y = train[['toxic','severe_toxic','obscene','threat','insult','identity_hate']].values
X_train, X_val, y_train, y_val =train_test_split(X, y, test_size=0.1, random_state=2020 )<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPool2D, BatchNormalization
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
if torch.cuda.is_available() :
device = torch.device("cuda")
 print(f'There are {torch.cuda.device_count() } GPU(s)available.')
 print('Device name:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )<categorify>
train_path = '.. /input/digit-recognizer/train.csv'
test_path = '.. /input/digit-recognizer/test.csv'
def text_preprocessing(text):

 text = re.sub(r'(@.*?)[\s]', ' ', text)
 text = re.sub(r'&amp;', '&', text)
 text = re.sub(r'\s+', ' ', text ).strip()
return text<train_model>
train_data = pd.read_csv(train_path)
 test_data = pd.read_csv(test_path )
print('Original:
', X[0])
 print('
Processed: ', text_preprocessing(X[0]))<load_pretrained>
img_rows, img_cols = 28, 28
num_classes = train_data.label.nunique()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case=True)
 def preprocessing_for_bert(data):

 input_ids = []
attention_masks = []
for sent in data:
encoded_sent = tokenizer.encode_plus(
 text = text_preprocessing(sent),
add_special_tokens = True,
max_length= MAX_LEN ,
pad_to_max_length = True,
return_attention_mask= True 
)
 input_ids.append(encoded_sent.get('input_ids'))
attention_masks.append(encoded_sent.get('attention_mask'))
input_ids = torch.tensor(input_ids)
 attention_masks = torch.tensor(attention_masks)
 return input_ids,attention_masks<define_variables>
def data_prep(raw_data, img_row, img_col, num_classes):
out_y = to_categorical(raw_data.label, num_classes)
 num_images = raw_data.shape[0]
x_as_array = raw_data.values[:, 1:]
x_as_matrix = x_as_array.reshape(num_images, img_row, img_col, 1)
 out_x = x_as_matrix/255.0
return out_x, out_y
all_text = np.concatenate([train.comment_text.values,test.comment_text.values])
 len_sent = [len(text_preprocessing(sent)) for sent in all_text]
avg_len = np.mean(len_sent)
 print('Avg length: ',avg_len )<define_variables>
X, y = data_prep(train_data, img_rows, img_cols, num_classes )
MAX_LEN = 150
token_ids = list(preprocessing_for_bert([X[0]])[0].squeeze().numpy())
 print('Original: ',X[0])
 print('Token IDs: ',token_ids )<categorify>
model = Sequential()
print('Tokenizing data...')
 train_inputs, train_masks = preprocessing_for_bert(X_train)
 val_inputs, val_masks = preprocessing_for_bert(X_val )<create_dataframe>
model.add(Conv2D(filters=40, kernel_size=(3,3), padding='Same', activation='relu', input_shape=(img_rows, img_cols, 1)))
 model.add(MaxPool2D(2,2))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters=60, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization(axis=1))
model.add(Dropout(0.25))
model.add(Conv2D(filters=80, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(BatchNormalization(axis=1))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(BatchNormalization(axis=1))
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization(axis=1))
model.add(Dropout(0.50))
model.add(BatchNormalization(axis=1))
model.add(Dense(10, activation='softmax'))
model.summary()
train_labels = torch.tensor(y_train)
 val_labels = torch.tensor(y_val)
 batch_size = 32
train_data = TensorDataset(train_inputs,train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 val_data = TensorDataset(val_inputs, val_masks, val_labels)
 val_sampler = SequentialSampler(val_data)
 val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size )<set_options>
model.compile(loss=categorical_crossentropy,
optimizer=Adam(0.001),
metrics=['accuracy'] )
%%time
class BertClassifier(nn.Module):

 def __init__(self, freeze_bert=False):

 super(BertClassifier,self ).__init__()
D_in, H, D_out = 768, 50 , 6
self.bert = BertModel.from_pretrained('bert-base-uncased')
 self.classifier = nn.Sequential(
 nn.Linear(D_in, H),
nn.ReLU() ,
nn.Linear(H, D_out))
if freeze_bert:
for param in self.bert.parameters() :
param.requires_grad = False
def forward(self,input_ids,attention_mask):

 outputs = self.bert(input_ids=input_ids,
attention_mask = attention_mask)
 last_hidden_state_cls = outputs[0][:,0,:]
logits = self.classifier(last_hidden_state_cls)
 return logits<choose_model_class>
def initialize_model(epochs=4):

 bert_classifier = BertClassifier(freeze_bert=False)
 bert_classifier.to(device)
 optimizer = AdamW(bert_classifier.parameters() ,
lr=5e-5,
eps=1e-8 
)
 total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
 return bert_classifier, optimizer, scheduler<choose_model_class>
history = model.fit(X, y, epochs=20, validation_split=0.2 )
loss_fn = nn.BCEWithLogitsLoss()
def set_seed(seed_value=42):

 random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):

 print("Start training...
 ")
 for epoch_i in range(epochs):
print(f"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}")
 print("-"*70)
 t0_epoch, t0_batch = time.time() , time.time()
total_loss, batch_loss, batch_counts = 0, 0, 0
model.train()
for step, batch in enumerate(train_dataloader):
batch_counts +=1
b_input_ids, b_attn_mask, b_labels = tuple(t.to(device)for t in batch)
 optimizer.zero_grad()
logits = model(b_input_ids, b_attn_mask)
 loss = loss_fn(logits, b_labels.float())
 batch_loss += loss.item()
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
if(step % 50000 == 0 and step != 0)or(step == len(train_dataloader)- 1):
time_elapsed = time.time() - t0_batch
print(f"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}")
 batch_loss, batch_counts = 0, 0
t0_batch = time.time()
avg_train_loss = total_loss / len(train_dataloader)
 print("-"*70)
 if evaluation == True:
val_loss, val_accuracy = evaluate(model, val_dataloader)
 time_elapsed = time.time() - t0_epoch
print(f"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}")
 print("-"*70)
 print("
")
 print("Training complete!")
 def evaluate(model, val_dataloader):

 model.eval()
val_accuracy = []
val_loss = []
for batch in val_dataloader:
b_input_ids, b_attn_mask, b_labels = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
logits = model(b_input_ids, b_attn_mask)
 loss = loss_fn(logits, b_labels.float())
 val_loss.append(loss.item())
 accuracy = accuracy_thresh(logits.view(-1,6),b_labels.view(-1,6))
val_accuracy.append(accuracy)
 val_loss = np.mean(val_loss)
 val_accuracy = np.mean(val_accuracy)
 return val_loss, val_accuracy
def accuracy_thresh(y_pred, y_true, thresh:float=0.5, sigmoid:bool=True):
"Compute accuracy when `y_pred` and `y_true` are the same size."
if sigmoid:
y_pred = y_pred.sigmoid()
return(( y_pred>thresh)==y_true.byte() ).float().mean().item() 
<train_model>
test_num_img = test_data.shape[0]
test_data_array = test_data.values
test_data_matrix = test_data_array.reshape(test_num_img, 28,28,1)
 test_data_matrix = test_data_matrix/255
set_seed(42)
 bert_classifier, optimizer, scheduler = initialize_model(epochs=1)
 train(bert_classifier, train_dataloader, val_dataloader, epochs=1, evaluation=True )<predict_on_test>
preds = model.predict_classes(test_data_matrix )
<predict_on_test><EOS> 
output = pd.DataFrame({'ImageId': list(range(1, len(preds)+1)) ,
'Label':preds})
 output.to_csv('output2.csv', index=False )