Kaggle competition: Kannada MNIST
gc.collect()<define_variables>
%matplotlib inline
batch_size = 256
def batch_gen(test_df):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr
all_preds = []
for x in tqdm(batch_gen(test_df)) :
all_preds.extend(model.predict(x ).flatten() )<predict_on_test>
print(( 'Start: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 print(( 'Data Load and Preprocessing: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 X_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 X_test = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 X_holdout = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv' )
%%time
val_preds = []
for x in batch_gen(val_df):
val_preds.extend(model.predict(x ).flatten() )<set_options>
id_train = X_train.index.values
y_train = X_train['label']
y_valid_pred = 0*y_train
X_train.drop(labels=['label'], axis=1, inplace=True)
 X_test.drop(labels=['id'], axis=1, inplace=True)
 X_test = X_test.astype('float32')/ 255.
 X_test = X_test.values.reshape(X_test.shape[0], 28, 28, 1 ).astype('float32')
 y_holdout = X_holdout['label']
X_holdout.drop(labels=['label'], axis=1, inplace=True)
 X_holdout = X_holdout.astype('float32')/ 255.
 X_holdout = X_holdout.values.reshape(X_holdout.shape[0], 28, 28, 1 ).astype('float32')
 my_init = 'glorot_uniform'
my_activ = 'relu'
my_optimiser = 'adam'
my_epsilon = 1e-8
nb_classes = 10
gc.collect()<categorify>
n_components = 30
pca = PCA(n_components=n_components ).fit(X_train.values)
 eigenvalues = pca.components_.reshape(n_components, 28, 28)
 eigenvalues = pca.components_
_thresh = []
for thresh in np.arange(0.1, 0.501, 0.01):
_thresh.append([thresh, f1_score(val_df["target"],(val_preds>thresh ).astype(int)) ])
 print("F1 score at threshold {0} is {1}".format(thresh, f1_score(val_df["target"],(val_preds>thresh ).astype(int))))<find_best_params>
def build_network(input_shape):
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(5, 5), strides=(1, 1), input_shape=input_shape, padding="same", activation='relu', kernel_initializer=my_init))
model.add(MaxPooling2D(pool_size=(2, 2), padding='valid'))
model.add(Dropout(0.25))
model.add(Conv2D(filters=128, kernel_size=(5, 5), strides=(1, 1), padding="same", activation='relu', kernel_initializer=my_init))
model.add(MaxPooling2D(pool_size=(2, 2), padding='valid'))
model.add(Dropout(0.25))
model.add(Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding="same", activation='relu', kernel_initializer=my_init))
model.add(MaxPooling2D(pool_size=(2, 2), padding='valid'))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, kernel_initializer=my_init))
model.add(Activation(my_activ))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(128, kernel_initializer=my_init))
model.add(Activation(my_activ))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(nb_classes, activation='softmax'))
model.compile(optimizer=my_optimiser,
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
_thresh = np.array(_thresh)
 best_id = _thresh[:,1].argmax()
best_thresh = _thresh[best_id][0]
best_thresh<save_to_csv>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=12,
zoom_range = 0.25,
shear_range=15,
width_shift_range=0.25,
height_shift_range=0.25,
horizontal_flip=False,
vertical_flip=False,
data_format="channels_last"
)
y_te =(np.array(all_preds)> best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<import_modules>
def epoch_cv(df_cv_per_epoch_val_acc, fold_num):
if fold_num==0:
df_cv_per_epoch_val_acc.index += 1
df_cv_per_epoch_val_acc['mean_val_acc'] = df_cv_per_epoch_val_acc.mean(axis=1)
 num_epochs = df_cv_per_epoch_val_acc.shape[0]
for i in range(1, num_epochs+1):
if i<moving_average_period+1:
df_cv_per_epoch_val_acc.at[i, 'moving_average'] = df_cv_per_epoch_val_acc.iloc[:i]['mean_val_acc'].mean()
else:
df_cv_per_epoch_val_acc.at[i, 'moving_average'] = df_cv_per_epoch_val_acc.iloc[i-moving_average_period:i]['mean_val_acc'].mean()
best_epoch_cv = df_cv_per_epoch_val_acc['mean_val_acc'].idxmax()
best_epoch_cv_by_moving_average = df_cv_per_epoch_val_acc['moving_average'].idxmax()
return df_cv_per_epoch_val_acc, best_epoch_cv, best_epoch_cv_by_moving_average
FOLD = 0
SEED = 1337
NOTIFY_EACH_EPOCH = False
WORKERS = 0
BATCH_SIZE = 512
N_SPLITS = 10
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def save_checkpoint(state):

 print(" Saving checkpoint")
 filename = f'./checkpoint-{state["epoch"]}.pt.tar'
torch.save(state, filename)
 def initialize(model, path=None, optimizer=None):
if path == None:
checkpoints = glob.glob('./*.pt.tar')
 path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0])for checkpoint in checkpoints])]
checkpoint = torch.load(path)
 model.load_state_dict(checkpoint['model'])
 print(f' Loaded checkpoint {path} | Trained for {checkpoint["epoch"] + 1} epochs')
 if optimizer:
optimizer.load_state_dict(checkpoint['optimizer'])
 epoch = checkpoint['epoch'] + 1
train_iteration = checkpoint['train_iteration']
val_iteration = checkpoint['val_iteration']
return model, optimizer, epoch, train_iteration, val_iteration
else:
return model<load_from_csv>
Run_CV = "Y"
Run_Holdout_Validation = "Y"
Run_Kaggle_Pred = "Y"
n_epochs = 75
my_batch_size = 128
my_verbose = 0
K = 4
len_test = len(X_test)
 moving_average_period = 10
mean_chart_lower_epoch_bound = 30
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<save_to_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=4, verbose=my_verbose, factor=0.55, min_lr=0.00001 )
kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 train_idx, val_idx = list(kfold.split(train)) [FOLD]
x_train, x_val = train.iloc[train_idx], train.iloc[val_idx]
x_train.to_csv('train.csv')
 x_val.to_csv('val.csv' )<string_transform>
print(" Epochs: " + str(n_epochs))
print(" Cross Validation Requested: " + Run_CV)
 print(" Kaggle Prediction Requested: " + Run_Kaggle_Pred )
nlp = English()
def tokenize(sentence):
x = nlp(sentence)
 return [token.text for token in x]<load_from_csv>
if Run_CV=="Y":
print(" Batch Size: " + str(my_batch_size))
print(" Number of K-Folds: " + str(K))
print(( 'Fold Preparation: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 kfold = StratifiedKFold(n_splits = K,
random_state = 2007,
shuffle = True)
 oof_pred = None
df_cv_per_epoch_train_acc = pd.DataFrame()
df_cv_per_epoch_val_acc = pd.DataFrame()
print(( 'KFold Model Starting: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 for i,(f_ind, outf_ind)in enumerate(kfold.split(X_train, y_train)) :
my_optimiser = Adam(lr=0.004, beta_1=0.9, beta_2=0.999, epsilon=my_epsilon, decay=0.0, amsgrad=False)
 X_train_f, X_val_f = X_train.loc[f_ind].copy() , X_train.loc[outf_ind].copy()
X_train_f = X_train_f.astype('float32')/ 255.
 X_train_f = X_train_f.values.reshape(X_train_f.shape[0], 28, 28, 1 ).astype('float32')
 X_val_f = X_val_f.astype('float32')/ 255.
 X_val_f = X_val_f.values.reshape(X_val_f.shape[0], 28, 28, 1 ).astype('float32')
 input_shape = X_train_f.shape[1:]
nnet_model = build_network(input_shape)
 y_train_f, y_val_f = y_train[f_ind], y_train[outf_ind]
y_train_f = y_train_f.values
y_val_f = y_val_f.values
y_val_f_series = y_val_f
y_train_f = to_categorical(y_train_f, num_classes = nb_classes)
 y_val_f = to_categorical(y_val_f, num_classes = nb_classes)
 print(( 'Augmenting Data: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 datagen.fit(X_train_f)
 print(( 'Model Fitting: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 print('Fold: ' + str(i))
history = nnet_model.fit_generator(datagen.flow(X_train_f,y_train_f, batch_size=my_batch_size), epochs=n_epochs, verbose=my_verbose,
steps_per_epoch=X_train.shape[0] // my_batch_size, validation_data=(X_val_f,y_val_f), callbacks=[learning_rate_reduction])
 df_cv_per_epoch_train_acc['fold_'+str(i)] = history.history['accuracy']
df_cv_per_epoch_val_acc['fold_'+str(i)] = history.history['val_accuracy']
y_preds = nnet_model.predict(X_val_f)
 if Run_Holdout_Validation=="Y":
if i==0:
holdout_preds = nnet_model.predict(X_holdout)
 else:
holdout_preds = holdout_preds + nnet_model.predict(X_holdout)
 if Run_Kaggle_Pred=="Y":
if i==0:
test_preds = nnet_model.predict(X_test)
 else:
test_preds = test_preds + nnet_model.predict(X_test)
 y_preds_series = np.argmax(y_preds,axis = 1)
 y_preds_series = pd.Series(y_preds_series,name="label")
 fold_accuracy = accuracy_score(y_val_f_series, y_preds_series)
 print(" Fold Accuracy = %3.6f"%(fold_accuracy))
if oof_pred is None:
oof_pred = y_preds_series
oof_pred_ids = outf_ind
else:
oof_pred = np.hstack(( oof_pred, y_preds_series))
oof_pred_ids = np.hstack(( oof_pred_ids, outf_ind))
df_cv_per_epoch_train_acc, best_train_epoch_cv, best_train_epoch_cv_by_moving_average = epoch_cv(df_cv_per_epoch_train_acc, i)
 df_cv_per_epoch_val_acc, best_epoch_cv, best_epoch_cv_by_moving_average = epoch_cv(df_cv_per_epoch_val_acc, i)
 print("Best Single Epoch: " + str(best_epoch_cv))
df_cv_per_epoch_val_acc['mean_train_acc'] = df_cv_per_epoch_train_acc['mean_val_acc']
PlotAcc(history, n_epochs)
 df_cv_per_epoch_val_acc.to_csv('df_cv_epoch_{:%Y%m%d%H%M%S}.csv'.format(datetime.datetime.now()), index=True)
 oof_pred = np.column_stack(( oof_pred_ids, oof_pred))
df_oof_pred = pd.DataFrame(oof_pred,index=oof_pred[:,0])
 df_oof_pred.columns = ['id', 'label']
df_oof_pred = df_oof_pred.sort_values(by=('id'), ascending=True)
 y_valid_pred = df_oof_pred['label'].values
oof_accuracy = accuracy_score(y_train, y_valid_pred)
 print(" Overall Out-of-Fold Accuracy = %3.4f"%(oof_accuracy))
else:
print("Cross-Validation skipped")
 if Run_Kaggle_Pred=="Y":
test_preds = test_preds / K
results = np.argmax(test_preds,axis = 1)
 results = pd.Series(results,name="label")
 print(( 'Writing Prediction: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 submission = pd.concat([pd.Series(range(0,len_test),name = "id"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )
question_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)
 target_field = data.Field(sequential=False, use_vocab=False, batch_first=True)
 train_fields = [ 
('id', None), 
('qid', None), 
('question_text', question_field), 
('target', target_field)
 ]
test_fields = [ 
('qid', None), 
('question_text', question_field)
 ]
train_dataset, val_dataset = data.TabularDataset.splits('./', train='train.csv', validation='val.csv', format='CSV', skip_header=True, fields=train_fields)
 test_dataset = data.TabularDataset('.. /input/test.csv', format='CSV', skip_header=True, fields=test_fields)
 vectors = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 question_field.build_vocab(train_dataset, max_size=95000)
 question_field.vocab.set_vectors(vectors.stoi, vectors.vectors, vectors.dim)
 pretrained_embedding = question_field.vocab.vectors
train_dataloader, val_dataloader = data.BucketIterator.splits(( train_dataset, val_dataset),(BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)
 test_dataloader = data.BucketIterator(test_dataset, 1, sort=False, shuffle=False)
 print(f'Train Dataset: {len(train_dataset)}')
 print(f'Val Dataset: {len(val_dataset)}')
 print(f'Test Dataset: {len(test_dataset)}' )<init_hyperparams>
lbound = min(n_epochs-1, mean_chart_lower_epoch_bound)
 ubound = max(n_epochs, lbound)+ 1
class SelfAttention(nn.Module):
def __init__(self, hidden_size, batch_first=False):
super(SelfAttention, self ).__init__()
self.hidden_size = hidden_size
self.batch_first = batch_first
self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)
 stdv = 1.0 / math.sqrt(self.hidden_size)
 for weight in self.att_weights:
nn.init.uniform_(weight, -stdv, stdv)
 def get_mask(self):
pass
def forward(self, inputs, lengths):
if self.batch_first:
batch_size, max_len = inputs.size() [:2]
else:
max_len, batch_size = inputs.size() [:2]
weights = torch.bmm(inputs,
self.att_weights 
.permute(1, 0)
.unsqueeze(0)
.repeat(batch_size, 1, 1)
)
 attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)
 mask = torch.ones(attentions.size() , requires_grad=True ).cuda()
for i, l in enumerate(lengths):
if l < max_len:
mask[i, l:] = 0
masked = attentions * mask
_sums = masked.sum(-1 ).unsqueeze(-1)
 attentions = masked.div(_sums)
 weighted = torch.mul(inputs, attentions.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations, attentions
class BaselineLSTM(nn.Module):
def __init__(self, embedding):
super(BaselineLSTM, self ).__init__()
self.embedding = nn.Embedding.from_pretrained(embedding)
 self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=2, batch_first=True, bidirectional=True)
 self.attention = SelfAttention(128*2, batch_first=True)
 self.fc = nn.Linear(128*2, 1)
 self.logit = nn.Linear(1, 1)
 def forward(self,x, x_len):
x = self.embedding(x)
 x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)
 out,(hidden, _)= self.lstm(x)
 x, lengths = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
 x, _ = self.attention(x, lengths)
 x = self.fc(x)
 x = self.logit(x ).view(-1)
 return x<choose_model_class>
if Run_Holdout_Validation=="Y":
holdout_preds = holdout_preds / K
results_holdout = np.argmax(holdout_preds,axis = 1)
 results_holdout = pd.Series(results_holdout,name="label")
 holdout_accuracy = accuracy_score(y_holdout, results_holdout)
 print(" Holdout Accuracy = %3.5f"%(holdout_accuracy))
start_epoch = 0
epochs = 7
early_stopping = 10
train_iteration = 0
val_iteration = 0
threshold = 0.35
model = BaselineLSTM(pretrained_embedding ).to(device)
 optimizer = optim.Adam(model.parameters())
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=True)
 criterion = nn.BCEWithLogitsLoss()
get_n_params(model )<train_model>
cv_weight = 7
holdout_weight = 1
blended_validation_accuracy =(( cv_weight * oof_accuracy)+(holdout_weight * holdout_accuracy)) /(cv_weight + holdout_weight)
 print("Blended Validation Accuracy = %3.5f"%(blended_validation_accuracy))
<predict_on_test><EOS> 
print(( 'Run Completed: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())) )