Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
print(os.listdir("/kaggle/input/digit-recognizer"))
N_FOLDS = 5
BATCH_SIZE = 256
def tokenize_encode(tweets, max_length=None):
return pretrained_bert_tokenizer(
 tweets,
add_special_tokens=True,
truncation=True,
padding="max_length",
max_length=max_length,
return_tensors="tf", 
)
 max_length_tweet = 72
max_length_keyword = 8
train_tweets_encoded = tokenize_encode(x_train["text"].to_list() , max_length_tweet)
 validation_tweets_encoded = tokenize_encode(x_val["text"].to_list() , max_length_tweet)
 train_keywords_encoded = tokenize_encode(x_train["keyword"].to_list() , max_length_keyword)
 validation_keywords_encoded = tokenize_encode(x_val["keyword"].to_list() , max_length_keyword)
 train_inputs_encoded = dict(train_tweets_encoded)
 train_inputs_encoded["keywords"] = train_keywords_encoded["input_ids"]
validation_inputs_encoded = dict(validation_tweets_encoded)
 validation_inputs_encoded["keywords"] = validation_keywords_encoded["input_ids"] 
<create_dataframe>
PATH = '/kaggle/input/digit-recognizer/'
train_dataset = tf.data.Dataset.from_tensor_slices(
(dict(train_tweets_encoded), y_train)
)
 val_dataset = tf.data.Dataset.from_tensor_slices(
(dict(validation_tweets_encoded), y_val)
)
 train_multi_input_dataset = tf.data.Dataset.from_tensor_slices(
(train_inputs_encoded, y_train)
)
 val_multi_input_dataset = tf.data.Dataset.from_tensor_slices(
(validation_inputs_encoded, y_val)
)
<feature_engineering>
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('Training on CPU...')
 else:
print('Training on GPU...' )
tfidf_vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(
 tokenizer=tweet_preprocessor, min_df=1, ngram_range=(1, 1), norm="l2" 
)
 train_vectors = tfidf_vectorizer.fit_transform(raw_documents=x_train["text"] ).toarray()
validation_vectors = tfidf_vectorizer.transform(x_val["text"] ).toarray()<train_model>
class DatasetMNIST(torch.utils.data.Dataset):
def __init__(self, data, augmentations=None):
self.data = data
self.augmentations = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
item = self.data.iloc[index]
image = item[1:].values.astype(np.uint8 ).reshape(( 28, 28, 1))
label = item[0]
if self.augmentations is not None:
augmented = self.augmentations(image=image)
 return augmented['image'], label
else:
return image, label
logisticRegressionClf = LogisticRegression(n_jobs=-1, C=2.78)
 logisticRegressionClf.fit(train_vectors, y_train)
 def print_metrics_sk(clf, x_train, y_train, x_val, y_val):
print(f"Train Accuracy: {clf.score(x_train, y_train):.2%}")
 print(f"Validation Accuracy: {clf.score(x_val, y_val):.2%}")
 print("")
 print(f"f1 score: {sklearn.metrics.f1_score(y_val, clf.predict(x_val)) :.2%}")
 print_metrics_sk(logisticRegressionClf, train_vectors, y_train, validation_vectors, y_val )<find_best_model_class>
dataset = pd.read_csv(f'{PATH}train.csv')
 dataset.head(1 )
feature_extractor = get_pretrained_bert_model()
model_outputs = feature_extractor.predict(
 train_dataset.batch(32)
)
 train_sentence_vectors = model_outputs.last_hidden_state[:, 0, :]
train_word_vectors = model_outputs.last_hidden_state[:, 1:, :]
model_outputs = feature_extractor.predict(
 val_dataset.batch(32)
)
 validation_sentence_vectors = model_outputs.last_hidden_state[:, 0, :]
validation_word_vectors = model_outputs.last_hidden_state[:, 1:, :]<train_model>
def custom_folds(dataset,n_folds=N_FOLDS):

 train_valid_id = []
start = 0
size = len(dataset)
 split = size // n_folds
valid_size = split
for i in range(n_folds):
train_data = dataset.drop(dataset.index[start:split] ).index.values
valid_data = dataset.loc[start:split-1].index.values
train_valid_id.append(( train_data,valid_data))
start += valid_size
split += valid_size
return train_valid_id
logisticRegressionClf = LogisticRegression(n_jobs=-1, class_weight=class_weights)
 logisticRegressionClf.fit(train_sentence_vectors, y_train)
 print_metrics_sk(
 logisticRegressionClf,
train_sentence_vectors,
y_train,
validation_sentence_vectors,
y_val,
)<train_on_grid>
train_valid = custom_folds(dataset=dataset )
def create_gru_model() -> keras.Model:
model = keras.Sequential()
model.add(keras.layers.InputLayer(input_shape=train_word_vectors.shape[1:]))
model.add(GRU(32, return_sequences=True))
model.add(GlobalMaxPooling1D())
 model.add(Dense(1, activation="sigmoid"))
model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_gru_model()
history = model.fit(
 train_word_vectors,
y_train,
validation_data=(validation_word_vectors, y_val),
class_weight=class_weights,
epochs=20,
verbose=0,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(model, train_word_vectors, y_train, validation_word_vectors, y_val )<choose_model_class>
transform_train = A.Compose([
A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=10),
A.Normalize(mean=(0.485,), std=(0.229,)) ,
ToTensor() ,
])
 transform_valid = A.Compose([
A.Normalize(mean=(0.485,), std=(0.229,)) ,
ToTensor() ,
] )
def create_multi_input_model() -> keras.Model:
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=feature_extractor.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(1 )(keyword_features)
 tweet_classification_vectors = keras.Input(( train_sentence_vectors.shape[1],), name="tweets")
 tweet_features = Dense(1, activation='relu' )(tweet_classification_vectors)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, tweet_classification_vectors], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_multi_input_model()
train_inputs = {"keywords" : train_keywords_encoded["input_ids"], "tweets" : train_sentence_vectors}
validation_inputs = {"keywords" : validation_keywords_encoded["input_ids"], "tweets" : validation_sentence_vectors}
history = model.fit(
 train_inputs,
y_train,
validation_data=(validation_inputs, y_val),
class_weight=class_weights,
epochs=20,
verbose=0,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(model, train_inputs, y_train, validation_inputs, y_val )<choose_model_class>
train_data = DatasetMNIST(dataset, augmentations=transform_train)
 valid_data = DatasetMNIST(dataset, augmentations=transform_valid)
 train_valid_loaders = []
for i in train_valid:
train_idx, valid_idx = i
train_sampler = SubsetRandomSampler(train_idx)
 valid_sampler = SubsetRandomSampler(valid_idx)
 train_loader = torch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE, sampler=train_sampler)
 valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=BATCH_SIZE, sampler=valid_sampler)
 train_valid_loaders.append(( train_loader,valid_loader))
def create_multi_input_rnn_model() -> keras.Model:
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=feature_extractor.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(1 )(keyword_features)
 tweet_token_embeddings = Input(train_word_vectors.shape[1:], name="tweets")
 tweet_features = GRU(32, return_sequences=True )(tweet_token_embeddings)
 tweet_features = GlobalMaxPooling1D()(tweet_features)
 tweet_features = Dense(1, activation='relu' )(tweet_features)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, tweet_token_embeddings], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
model = create_multi_input_rnn_model()
train_inputs = {"keywords" : train_keywords_encoded["input_ids"], "tweets" : train_word_vectors}
validation_inputs = {"keywords" : validation_keywords_encoded["input_ids"], "tweets" : validation_word_vectors}
history = model.fit(
 train_inputs,
y_train,
validation_data=(validation_inputs, y_val),
class_weight=class_weights,
epochs=20,
verbose=0,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=5,
restore_best_weights=True, 
)
 ], 
)
 print_metrics(model, train_inputs, y_train, validation_inputs, y_val )<choose_model_class>
class Net(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 32, kernel_size=3)
 self.bn1 = nn.BatchNorm2d(32)
 self.conv2 = nn.Conv2d(32, 32, kernel_size=3)
 self.bn2 = nn.BatchNorm2d(32)
 self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2, padding=2)
 self.bn3 = nn.BatchNorm2d(32)
 self.conv4 = nn.Conv2d(32, 64, kernel_size=3)
 self.bn4 = nn.BatchNorm2d(64)
 self.conv5 = nn.Conv2d(64, 64, kernel_size=3)
 self.bn5 = nn.BatchNorm2d(64)
 self.conv6 = nn.Conv2d(64, 64, kernel_size=5, stride=2, padding=2)
 self.bn6 = nn.BatchNorm2d(64)
 self.conv7 = nn.Conv2d(64, 128, kernel_size=4)
 self.bn7 = nn.BatchNorm2d(128)
 self.lin1 = nn.Linear(128,10)
 def forward(self, xb):
x = xb.view(-1, 1, 28, 28)
 x = self.bn1(F.relu(self.conv1(x)))
 x = self.bn2(F.relu(self.conv2(x)))
 x = self.bn3(F.relu(self.conv3(x)))
 x = F.dropout2d(x, 0.25)
 x = self.bn4(F.relu(self.conv4(x)))
 x = self.bn5(F.relu(self.conv5(x)))
 x = self.bn6(F.relu(self.conv6(x)))
 x = F.dropout2d(x, 0.25)
 x = self.bn7(F.relu(self.conv7(x)))
 x = torch.flatten(x, start_dim=1)
 x = F.dropout2d(x, 0.25)
 x = self.lin1(x)
 x = F.softmax(x, dim=1)
 return x
model = Net()
print(model)
 if train_on_gpu:
model.cuda()
def create_candidate_model_with_fx(hp: kerastuner.HyperParameters)-> keras.Model:
keyword_ids = keras.Input(( 8,), name="keywords")
 keyword_features = Embedding(input_dim=feature_extractor.config.vocab_size, output_dim=16, input_length=8, mask_zero=True )(keyword_ids)
 keyword_features = Flatten()(keyword_features)
 keyword_features = Dense(hp.Choice("keyword_units", values=[1, 8, 16, 32], default=1))(keyword_features)
 tweet_token_embeddings = Input(train_word_vectors.shape[1:], name="tweets")
 tweet_features = GRU(hp.Choice("GRU_units", values=[8, 16, 32, 64, 128], default=32), return_sequences=True )(tweet_token_embeddings)
 tweet_features = Dropout(hp.Float("GRU_dropout", min_value=0.0, max_value=0.5, step=0.1))(tweet_features)
 tweet_features = GlobalMaxPooling1D()(tweet_features)
 for i in range(hp.Int("num_layers", min_value=0, max_value=3, step=1)) :
tweet_features = Dense(hp.Choice("layer_" + str(i)+ "_units", values=[2, 8, 16, 32, 64, 128, 256]), activation="relu" )(tweet_features)
 tweet_features = Dropout(hp.Float("layer_" + str(i)+ "_dropout", min_value=0.0, max_value=0.5, step=0.1))(tweet_features)
 combined_features = concatenate([keyword_features, tweet_features])
 combined_prediction = Dense(1, activation="sigmoid" )(combined_features)
 model = keras.Model(inputs = [keyword_ids, tweet_token_embeddings], outputs=combined_prediction)
 model.compile(
 optimizer=keras.optimizers.Adam() ,
loss="binary_crossentropy",
metrics=keras.metrics.BinaryAccuracy(name="accuracy"), 
)
 return model
train_inputs = {"keywords" : train_keywords_encoded["input_ids"], "tweets" : train_word_vectors}
validation_inputs = {"keywords" : validation_keywords_encoded["input_ids"], "tweets" : validation_word_vectors} 
<define_variables>
class DatasetSubmissionMNIST(torch.utils.data.Dataset):
def __init__(self, file_path, augmentations=None):
self.data = pd.read_csv(file_path)
 self.augmentations = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
image = self.data.iloc[index].values.astype(np.uint8 ).reshape(( 28, 28, 1))
if self.augmentations is not None:
augmented = self.augmentations(image=image)
 return augmented['image']
return image
MAX_EPOCHS = 10
FACTOR = 3
ITERATIONS = 3
print(f"Number of models in each bracket: {math.ceil(1 + math.log(MAX_EPOCHS, FACTOR)) }")
 print(f"Number of epochs over all trials: {round(ITERATIONS *(MAX_EPOCHS *(math.log(MAX_EPOCHS, FACTOR)** 2)))}" )<train_on_grid>
transform_test = A.Compose([
A.Normalize(mean=(0.485,), std=(0.229,)) ,
ToTensor() ,
])
 submissionset = DatasetSubmissionMNIST(f'{PATH}test.csv', augmentations=transform_test)
 submissionloader = torch.utils.data.DataLoader(submissionset, batch_size=BATCH_SIZE, shuffle=False )
tuner = kerastuner.Hyperband(
 create_candidate_model_with_fx,
max_epochs=MAX_EPOCHS,
hyperband_iterations=ITERATIONS,
factor=FACTOR,
objective="val_accuracy",
directory="hyperparam-search",
project_name="architecture-hyperband", 
)
 tuner.search(
 train_inputs,
y_train,
validation_data=(validation_inputs, y_val),
class_weight=class_weights,
epochs=10,
verbose=1,
callbacks=[
EarlyStopping(
 monitor="val_accuracy",
min_delta=0.001,
patience=3,
restore_best_weights=True, 
)
 ], 
)
<train_model>
def every_predict(model,submissionloader=submissionloader):
all_batchs = []
with torch.no_grad() :
model.eval()
for images in submissionloader:
if train_on_gpu:
images = images.cuda()
ps = model(images)
 all_batchs.append(ps.to('cpu' ).detach().numpy())
 return all_batchs
best_model = tuner.get_best_models() [0]
print("")
 best_arch_hp = tuner.get_best_hyperparameters() [0]
pprint.pprint(best_arch_hp.values, indent=4)
 print("")
 print_metrics(best_model, train_inputs, y_train, validation_inputs, y_val )<choose_model_class>
five_predict = []
all_train_losses, all_valid_losses = [], []
FOLD = 1
for i in train_valid_loaders:
model = Net()
if train_on_gpu:
model.cuda()
train_loader, valid_loader = i
LEARNING_RATE = 0.01
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters() ,lr=LEARNING_RATE)
 epochs = 120
valid_loss_min = np.Inf
train_losses, valid_losses = [], []
history_accuracy = []
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,factor=0.7,patience=2)
 model.train()
for e in range(1, epochs+1):
running_loss = 0
for images, labels in train_loader:
if train_on_gpu:
images, labels = images.cuda() , labels.cuda()
optimizer.zero_grad()
ps = model(images)
 loss = criterion(ps, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
else:
valid_loss = 0
accuracy = 0
with torch.no_grad() :
model.eval()
for images, labels in valid_loader:
if train_on_gpu:
images, labels = images.cuda() , labels.cuda()
ps = model(images)
 _, top_class = ps.topk(1, dim=1)
 equals = top_class == labels.view(*top_class.shape)
 valid_loss += criterion(ps, labels)
 accuracy += torch.mean(equals.type(torch.FloatTensor))
model.train()
train_losses.append(running_loss/len(train_loader))
valid_losses.append(valid_loss/len(valid_loader))
history_accuracy.append(accuracy/len(valid_loader))
network_learned = valid_loss < valid_loss_min
if e == 1 or e % 5 == 0 or network_learned:
print(f"Epoch: {e}/{epochs}.. ",
f"Training Loss: {running_loss/len(train_loader):.4f}.. ",
f"Validation Loss: {valid_loss/len(valid_loader):.4f}.. ",
f"Valid Accuracy: {accuracy/len(valid_loader):.4f}")
 if network_learned:
valid_loss_min = valid_loss
torch.save(model.state_dict() , f'best_model_fold{FOLD}.pt')
 print('Detected network improvement, saving current model')
 scheduler.step(running_loss)
 all_train_losses.append(train_losses)
 all_valid_losses.append(valid_losses)
 model.load_state_dict(torch.load(f'best_model_fold{FOLD}.pt'))
model.eval()
five_predict.append(every_predict(model))
model.train()
FOLD +=1
<choose_model_class><EOS> 
flat_list = []
for sublist in five_predict:
for item in sublist:
for i in item:
flat_list.append(i)
 final = []
for i in range(0,28000):
numbers = [i+a*28000 for a in range(N_FOLDS)]
final.append(sum(flat_list[C] for C in numbers))
subm = np.argmax(( final),axis=1)
 sample_subm = pd.read_csv(f'{PATH}sample_submission.csv')
 sample_subm['Label'] = subm
sample_subm.to_csv('submission.csv',index=False )