Kaggle competition: Digit Recognizer
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original sentence: ', sentences[0])
 print('Token IDs list:', input_ids[0])
<split>
%matplotlib inline
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(0.8 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))<load_pretrained>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
batch_size = 32
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)<load_pretrained>
x_train = np.array(x_train)
 test = np.array(test )
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()<choose_model_class>
y_train = to_categorical(y_train,10 )
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8 
)
 epochs = 4
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )<compute_test_metric>
x_train, x_val, y_train, y_val=train_test_split(x_train,y_train,test_size=0.1 )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat)
 def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<train_model>
model = Sequential()
training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took {:} time".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )<categorify>
model.add(Input(shape=(28, 28, 1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(1,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(MaxPooling2D(pool_size=(1,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(128,(2,2),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(128,(2,2),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
print('Number of test sentences: {:,}
'.format(test.shape[0]))
sentences_test = test.text.values
input_ids = []
attention_masks = []
for sent in sentences_test:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 batch_size = batch_size
prediction_data = TensorDataset(input_ids, attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )<predict_on_test>
model.compile(loss=keras.losses.categorical_crossentropy,\
optimizer = tf.keras.optimizers.Adam() ,\
metrics=['accuracy'] )
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
predictions.append(logits)
 print('DONE!')
<concatenate>
model.fit(x_train,y_train, batch_size = 128, epochs = 30, validation_data=(x_val,y_val))
all_logits = torch.cat(predictions, dim=0)
 probs = F.softmax(all_logits, dim=1 ).cpu().numpy()
probs<prepare_output>
y_pred = model.predict(x_val)
 y_pred_classes = np.argmax(y_pred,axis = 1)
 y_pred_classes
y_true = np.argmax(y_val,axis = 1)
 y_true
confusion_mtx = confusion_matrix(y_true, y_pred_classes)
 confusion_mtx
threshold = 0.5
preds = np.where(probs[:, 1] > threshold, 1, 0)
 preds<count_values>
model.evaluate(x_val,y_val,verbose=0 )
print("Number of tweets labeled as true disaster tweet: ", preds.sum() )<prepare_x_and_y>
train_image_generator = ImageDataGenerator()
Y_test = preds<save_to_csv>
history = model.fit_generator(train_image_generator.flow(x_train,y_train, batch_size =32),epochs = 3,validation_data =(x_val,y_val))
df_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv', index_col=0 ).fillna('')
 df_submission['target'] = Y_test
df_submission.to_csv('submission.csv')
 !head submission.csv<set_options>
y_pred = model.predict(x_val)
 y_pred_classes = np.argmax(y_pred,axis = 1)
 y_pred_classes
warnings.filterwarnings('ignore')
 stop = set(stopwords.words('english'))
%matplotlib inline
plt.style.use('ggplot')
<load_from_csv>
y_true = np.argmax(y_val,axis = 1)
 y_true
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 train_sent, test_sent, train_label = train.text.values, test.text.values, train.target.values<train_model>
confusion_mtx = confusion_matrix(y_true, y_pred_classes)
 confusion_mtx
word_tokenizer = Tokenizer()
word_tokenizer.fit_on_texts(train_sent)
 vocab_length = len(word_tokenizer.word_index)+ 1<string_transform>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<feature_engineering><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )