Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
seed = 13
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )<load_from_csv>
df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 df_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
def get_data() :
print('getting test and train data...')
 path = '.. /input/google-quest-challenge/'
train = pd.read_csv(path+'train.csv')
 test = pd.read_csv(path+'test.csv')
 submission = pd.read_csv(path+'sample_submission.csv')
 y = train[train.columns[11:]]
X = train[['question_title', 'question_body', 'answer']]
X_test = test[['question_title', 'question_body', 'answer']]
X.question_body = X.question_body.apply(html.unescape)
 X.question_title = X.question_title.apply(html.unescape)
 X.answer = X.answer.apply(html.unescape)
 X_test.question_body = X_test.question_body.apply(html.unescape)
 X_test.question_title = X_test.question_title.apply(html.unescape)
 X_test.answer = X_test.answer.apply(html.unescape)
 return X, X_test, y, train, test<load_pretrained>
X_train = df_train.drop('label', axis=1 ).values
y_train = df_train['label'].values.reshape(-1,1)
 X_test = df_test.values
def get_tokenizer(model_name):
print(f'getting tokenizer for {model_name}...')
 if model_name == 'xlnet-base-cased':
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
 elif model_name == 'roberta-base':
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 elif model_name == 'bert-base-uncased':
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 return tokenizer<categorify>
X_train = X_train.astype('float32')/255.0
X_test = X_test.astype('float32')/255.0
def fix_length(tokens, max_sequence_length=512, q_max_len=254, a_max_len=254, model_type='questions'):
if model_type == 'questions':
length = len(tokens)
 if length > max_sequence_length:
tokens = tokens[:max_sequence_length-1]
return tokens
else:
question_tokens, answer_tokens = tokens
q_len = len(question_tokens)
 a_len = len(answer_tokens)
 if q_len + a_len + 3 > max_sequence_length:
if a_max_len <= a_len and q_max_len <= q_len:
q_new_len_head = q_max_len//2
question_tokens = question_tokens[:q_new_len_head] + question_tokens[-q_new_len_head:]
a_new_len_head = a_max_len//2
answer_tokens = answer_tokens[:a_new_len_head] + answer_tokens[-a_new_len_head:]
elif q_len <= a_len and q_len < q_max_len:
a_max_len = a_max_len +(q_max_len - q_len - 1)
 a_new_len_head = a_max_len//2
answer_tokens = answer_tokens[:a_new_len_head] + answer_tokens[-a_new_len_head:]
elif a_len < q_len:
q_max_len = q_max_len +(a_max_len - a_len - 1)
 q_new_len_head = q_max_len//2
question_tokens = question_tokens[:q_new_len_head] + question_tokens[-q_new_len_head:]
return question_tokens, answer_tokens<categorify>
X_train = X_train.reshape(-1,28,28,1)
 X_test = X_test.reshape(-1,28,28,1)
 y_train = to_categorical(y_train)
 target_count = y_train.shape[1]
def transformer_inputs(title, question, answer, tokenizer, model_type='questions', MAX_SEQUENCE_LENGTH = 512):
if model_type == 'questions':
question = f"{title} [SEP] {question}"
question_tokens = tokenizer.tokenize(question)
 question_tokens = fix_length(question_tokens, model_type=model_type)
 ids_q = tokenizer.convert_tokens_to_ids(["[CLS]"] + question_tokens)
 padded_ids =(ids_q + [tokenizer.pad_token_id] *(MAX_SEQUENCE_LENGTH - len(ids_q)))[:MAX_SEQUENCE_LENGTH]
token_type_ids =([0] * MAX_SEQUENCE_LENGTH)[:MAX_SEQUENCE_LENGTH]
attention_mask =([1] * len(ids_q)+ [0] *(MAX_SEQUENCE_LENGTH - len(ids_q)))[:MAX_SEQUENCE_LENGTH]
return padded_ids, token_type_ids, attention_mask
else:
question = f"{title} [SEP] {question}"
question_tokens = tokenizer.tokenize(question)
 answer_tokens = tokenizer.tokenize(answer)
 question_tokens, answer_tokens = fix_length(tokens=(question_tokens, answer_tokens), model_type=model_type)
 ids = tokenizer.convert_tokens_to_ids(["[CLS]"] + question_tokens + ["[SEP]"] + answer_tokens + ["[SEP]"])
 padded_ids = ids + [tokenizer.pad_token_id] *(MAX_SEQUENCE_LENGTH - len(ids))
token_type_ids = [0] *(1 + len(question_tokens)+ 1)+ [1] *(len(answer_tokens)+ 1)+ [0] *(MAX_SEQUENCE_LENGTH - len(ids))
attention_mask = [1] * len(ids)+ [0] *(MAX_SEQUENCE_LENGTH - len(ids))
return padded_ids, token_type_ids, attention_mask<categorify>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
def input_data(df, tokenizer, model_type='questions'):
print(f'generating {model_type} input for transformer...')
 input_ids, input_token_type_ids, input_attention_masks = [], [], []
for title, body, answer in tqdm(zip(df["question_title"].values, df["question_body"].values, df["answer"].values)) :
ids, type_ids, mask = transformer_inputs(title, body, answer, tokenizer, model_type=model_type)
 input_ids.append(ids)
 input_token_type_ids.append(type_ids)
 input_attention_masks.append(mask)
 return(
 np.asarray(input_ids, dtype=np.int32),
np.asarray(input_attention_masks, dtype=np.int32),
np.asarray(input_token_type_ids, dtype=np.int32))<load_pretrained>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42 )
def get_model(name):
if name == 'xlnet-base-cased':
config = XLNetConfig.from_pretrained('xlnet-base-cased', output_hidden_states=True)
 model = TFXLNetModel.from_pretrained('xlnet-base-cased', config=config)
 elif name == 'roberta-base':
config = RobertaConfig.from_pretrained('roberta-base', output_hidden_states=True)
 model = TFRobertaModel.from_pretrained('roberta-base', config=config)
 elif name == 'bert-base-uncased':
config = BertConfig.from_pretrained('bert-base-uncased', output_hidden_states=True)
 model = TFBertModel.from_pretrained('bert-base-uncased', config=config)
 return model<choose_model_class>
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(BatchNormalization())
 model.add(Dense(target_count, activation='softmax'))
optimizer = RMSprop(learning_rate=0.001,rho=0.99)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.3, verbose=1,patience=2, min_lr=0.00000001)
 callback = EarlyStopping(monitor='loss', patience=5)
 history = model.fit(datagen.flow(X_train,y_train, batch_size=64), epochs = 50, validation_data=(X_val, y_val), verbose = 1, callbacks=[reduce_lr, callback] )
def create_model(name='xlnet-base-cased', model_type='questions'):
print(f'creating model {name}...')
 K.clear_session()
max_seq_length = 512
input_tokens = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name="input_tokens")
 input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name="input_mask")
 input_segment = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name="input_segment")
 model = get_model(name)
 if(name == 'xlnet-base-cased'):
sequence_output, hidden_states = model([input_tokens, input_mask])
 elif(name=='roberta-base' and model_type!='questions'):
sequence_output, pooler_output, hidden_states = model([input_tokens, input_mask])
 else:
sequence_output, pooler_output, hidden_states = model([input_tokens, input_mask, input_segment])
 h12 = tf.reshape(hidden_states[-1][:,0],(-1,1,768))
h11 = tf.reshape(hidden_states[-2][:,0],(-1,1,768))
h10 = tf.reshape(hidden_states[-3][:,0],(-1,1,768))
h09 = tf.reshape(hidden_states[-4][:,0],(-1,1,768))
concat_hidden = tf.keras.layers.Concatenate(axis=2 )([h12, h11, h10, h09])
 x = GlobalAveragePooling1D()(concat_hidden)
 x = Dropout(0.2 )(x)
 if model_type == 'answers':
output = Dense(9, activation='sigmoid' )(x)
 elif model_type == 'questions':
output = Dense(21, activation='sigmoid' )(x)
 else:
output = Dense(30, activation='sigmoid' )(x)
 if(name == 'xlnet-base-cased')or(name=='roberta-base' and model_type!='questions'):
model = Model(inputs=[input_tokens, input_mask], outputs=output)
 else:
model = Model(inputs=[input_tokens, input_mask, input_segment], outputs=output)
 return model<categorify>
y_val_m = y_val.argmax(axis=1)
 y_val_hat_prob = model.predict(X_val)
 y_val_hat = y_val_hat_prob.argmax(axis=1)
 X_val_inc = X_val[y_val_m != y_val_hat, :, :, :]
y_val_inc = y_val_m[y_val_m != y_val_hat]
y_val_hat_inc = y_val_hat[y_val_m != y_val_hat]
y_val_hat_prob_inc = y_val_hat_prob[y_val_m != y_val_hat]
class data_generator:
def __init__(self, X, X_test, tokenizer, type_):
tokens, masks, segments = input_data(X_test, tokenizer, type_)
 self.test_data = {'input_tokens': tokens,
'input_mask': masks,
'input_segment': segments}
self.tokens, self.masks, self.segments = input_data(X, tokenizer, type_)
 def generate_data(tr, cv, name='xlnet-base-cased', model_type='questions'):
if name!='xlnet-base-cased':
train_data = {'input_tokens': self.tokens[tr],
'input_mask': self.masks[tr],
'input_segment': self.segments[tr]}
cv_data = {'input_tokens': self.tokens[cv],
'input_mask': self.masks[cv],
'input_segment': self.segments[cv]}
else:
train_data = {'input_tokens': self.tokens[tr],
'input_mask': self.masks[tr]}
cv_data = {'input_tokens': self.tokens[cv],
'input_mask': self.masks[cv]}
if model_type=='questions':
y_tr = y.values[tr, 21:]
y_cv = y.values[cv, 21:]
elif model_type=='answers':
y_tr = y.values[tr, 21:]
y_cv = y.values[cv, 21:]
else:
y_tr = y.values[tr]
y_cv = y.values[cv]
return train_data, cv_data, y_tr, y_cv<categorify>
for i in range(0,10):
act = y_val_inc[i]
pred = y_val_hat_inc[i]
print('Actual: {}; Confidence(act/pred): \t{} - {:.0f}% \t{} - {:.0f}%'.format(act, act, y_val_hat_prob_inc[i][act]*100, pred, y_val_hat_prob_inc[i][pred]*100))
def optimize_ranks(preds, unique_labels):
print(f'optimizing the predicted values...')
 new_preds = np.zeros(preds.shape)
 for i in range(preds.shape[1]):
interpolate_bins = np.digitize(preds[:, i], bins=unique_labels, right=False)
 if len(np.unique(interpolate_bins)) == 1:
new_preds[:, i] = preds[:, i]
else:
new_preds[:, i] = unique_labels[interpolate_bins]
return new_preds<count_unique_values>
y_test_hat = model.predict(X_test ).argmax(axis=1)
 df_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 df_submission['Label'] = y_test_hat.astype('int32')
 df_submission.to_csv('Submission.csv', index=False)
 print('Submission saved!' )
def get_exp_labels(train):
X = train.iloc[:, 11:]
unique_labels = np.unique(X.values)
 denominator = 60
q = np.arange(0, 101, 100 / denominator)
 exp_labels = np.percentile(unique_labels, q)
 return exp_labels<compute_test_metric>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )<compute_test_metric>
df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 df_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
def rhos(y, y_pred):
return tf.py_function(compute_spearmanr_ignore_nan,(y, y_pred), tf.double )<train_model>
X_train = df_train.drop('label', axis=1 ).values
y_train = df_train['label'].values.reshape(-1,1)
 X_test = df_test.values
def fit_model(model, model_name, model_type, data_gen, file_path, train, use_saved_weights=True):
path = '.. /input/google-qna-predicted-data/'
if use_saved_weights:
print(f'getting saved weights for {model_name}...')
 model.load_weights(path+file_path)
 else:
print(f'fitting data on {model_name}...')
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.00002)
 model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=[rhos])
 kf = KFold(n_splits=5, random_state=42)
 for tr, cv in kf.split(np.arange(train.shape[0])) :
tr_data, cv_data, y_tr, y_cv = data_gen.generate_data(tr, cv, model_name, model_type)
 model.fit(tr_data, y_tr, epochs=1, batch_size=4, validation_data=(cv_data, y_cv))
model.save_weights(file_path)
 return model<concatenate>
X_train = X_train.astype('float32')/255.0
X_test = X_test.astype('float32')/255.0
def get_weighted_avg(model_predictions):
xlnet_q, xlnet_a, roberta_q, roberta_a, roberta_qa, bert_q, bert_a, bert_qa = model_predictions
xlnet_concat = np.concatenate(( xlnet_q, xlnet_a), axis=1)
 bert_concat = np.concatenate(( bert_q, bert_a), axis=1)
 roberta_concat = np.concatenate(( roberta_q, roberta_a), axis=1)
 predict =(roberta_qa + bert_qa + xlnet_concat + bert_concat + roberta_concat)/5
return predict<define_variables>
X_train = X_train.reshape(-1,28,28,1)
 X_test = X_test.reshape(-1,28,28,1)
 y_train = to_categorical(y_train)
 target_count = y_train.shape[1]
def get_predictions(predictions_present=True, model_saved_weights_present=True):
msw = model_saved_weights_present
X, X_test, y, train, test = get_data()
path = '.. /input/google-qna-predicted-data/'
model_names = ['xlnet-base-cased', 'roberta-base', 'bert-base-uncased']
model_types = ['questions', 'answers', 'questions_answers']
saved_weights_names = ['xlnet_q.h5', 'xlnet_a.h5', 'roberta_q.h5', 'roberta_a.h5',
'roberta_qa.h5', 'bert_q.h5', 'bert_a.h5', 'bert_qa.h5']
saved_model_predictions = [path+'xlnet_q.csv', path+'xlnet_a.csv', path+'roberta_q.csv', path+'roberta_a.csv',
path+'roberta_qa.csv', path+'bert_q.csv', path+'bert_a.csv', path+'bert_qa.csv']
model_predictions = []
if predictions_present:
model_predictions = [pd.read_csv(file_name ).values for file_name in saved_model_predictions]
else:
i = 0
for name_ in model_names:
for type_ in model_types:
if name_ == 'xlnet-base-cased' and type_ == 'questions_answers':
continue
print('-'*100)
 model = create_model(name_, type_)
 tokenizer = get_tokenizer(name_)
 data_gen = data_generator(X, X_test, tokenizer, type_)
 model = fit_model(model, name_, type_, data_gen, saved_weights_names[i], train, msw)
 print(f'getting target predictions from {name_}...')
 model_predictions.append(model.predict(data_gen.test_data))
i+=1
predicted_labels = get_weighted_avg(model_predictions)
 exp_labels = get_exp_labels(train)
 optimized_predicted_labels = optimize_ranks(predicted_labels, exp_labels)
 df = pd.concat([test['qa_id'], pd.DataFrame(optimized_predicted_labels, columns=train.columns[11:])], axis=1)
 print('done...!')
 return df<save_to_csv>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
submission = get_predictions(predictions_present=True)
<save_to_csv>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42 )
submission = pd.read_csv('.. /input/google-qna-predicted-data/output.csv')
 sample_submission = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv')
 sample_submission = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv')
 id_in_sub = set(submission.qa_id)
 id_in_sample_submission = set(sample_submission.qa_id)
 diff = id_in_sample_submission - id_in_sub
sample_submission = pd.concat([submission, sample_submission[sample_submission.qa_id.isin(diff)]] ).reset_index(drop=True)
 sample_submission.to_csv("submission.csv", index=False )<set_options>
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(BatchNormalization())
 model.add(Dense(target_count, activation='softmax'))
optimizer = RMSprop(learning_rate=0.001,rho=0.99)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.3, verbose=1,patience=2, min_lr=0.00000001)
 callback = EarlyStopping(monitor='loss', patience=5)
 history = model.fit(datagen.flow(X_train,y_train, batch_size=64), epochs = 50, validation_data=(X_val, y_val), verbose = 1, callbacks=[reduce_lr, callback] )
%matplotlib inline 
<load_from_csv>
y_val_m = y_val.argmax(axis=1)
 y_val_hat_prob = model.predict(X_val)
 y_val_hat = y_val_hat_prob.argmax(axis=1)
 X_val_inc = X_val[y_val_m != y_val_hat, :, :, :]
y_val_inc = y_val_m[y_val_m != y_val_hat]
y_val_hat_inc = y_val_hat[y_val_m != y_val_hat]
y_val_hat_prob_inc = y_val_hat_prob[y_val_m != y_val_hat]
train_df = pd.read_csv(".. /input/GiveMeSomeCredit/cs-training.csv")
 test_df = pd.read_csv(".. /input/GiveMeSomeCredit/cs-test.csv")
 combine = [train_df, test_df]<create_dataframe>
for i in range(0,10):
act = y_val_inc[i]
pred = y_val_hat_inc[i]
print('Actual: {}; Confidence(act/pred): \t{} - {:.0f}% \t{} - {:.0f}%'.format(act, act, y_val_hat_prob_inc[i][act]*100, pred, y_val_hat_prob_inc[i][pred]*100))
<count_missing_values><EOS> 
y_test_hat = model.predict(X_test ).argmax(axis=1)
 df_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 df_submission['Label'] = y_test_hat.astype('int32')
 df_submission.to_csv('Submission.csv', index=False)
 print('Submission saved!' )