Kaggle competition: Digit Recognizer
estimators = [ 
('lgb', lgb.LGBMClassifier(n_estimators=54)) , 
('rfc', RandomForestClassifier(n_estimators=200)) , 
('mlp', MLPClassifier(hidden_layer_sizes=2)) , 
('knn', KNeighborsClassifier(n_neighbors=320, weights='distance', algorithm='auto'))
]<train_model>
%matplotlib inline
best_l, best_pt, maxauc = 0, 'none', 0
for hid_lay_siz in [1,2,3,4,5]:
for pass_throu in [True, False]:
reg = StackingClassifier(
 estimators=estimators,
final_estimator=MLPClassifier(
 hidden_layer_sizes=hid_lay_siz,
random_state=0 
),
passthrough=pass_throu,
verbose=3 
)
 reg.fit(X_learn, y_learn)
 y_pred = reg.predict_proba(X_valid)[:,1]
score = roc_auc_score(y_valid, y_pred)
 print(score)
 if score > maxauc:
best_l, best_pt, maxauc = hid_lay_siz, pass_throu, score
print()
print(best_l, best_pt, maxauc)
<train_model>
img_rows, img_cols = 28, 28
reg = StackingClassifier(
 estimators=estimators,
final_estimator=MLPClassifier(
 hidden_layer_sizes=best_l,
random_state=0 
),
passthrough=best_pt,
verbose=3 
)
 reg.fit(X_train, y_train)
 y_pred = reg.predict_proba(X_test)[:,1] 
<load_from_csv>
df_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 df_test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 df_train.head()
sample = pd.read_csv('.. /input/GiveMeSomeCredit/sampleEntry.csv')
 sample<save_to_csv>
y_train = df_train["label"]
X_train = df_train.drop(labels = ["label"],axis = 1)
 X_test = df_test
X_train /= 255
X_test /= 255
X_train = X_train.values.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_test = X_test.values.reshape(X_test.shape[0], img_rows, img_cols, 1)
 X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=1)
 y_train = to_categorical(y_train, num_classes=10)
 y_val = to_categorical(y_val, num_classes=10 )
sample['Probability'] = y_pred
sample.to_csv('./submit.csv',index=False)
 reload = pd.read_csv('./submit.csv')
 reload 
<set_options>
batch_size = 128
epochs = 200
lr = 1e-4
rho = 0.7
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.000001 )
np.set_printoptions(suppress=True)
 print(tf.__version__ )<load_from_csv>
input_shape =(img_rows, img_cols, 1)
 num_classes = 10
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.compile(loss=categorical_crossentropy,
optimizer=RMSprop(learning_rate=lr, rho=rho),
metrics=['accuracy'] )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 384
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<categorify>
model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_val, y_val))
score = model.evaluate(X_val, y_val, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<feature_engineering>
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def create_model() :
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)
 return model<categorify>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )