Kaggle competition: Leaf Classification
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<load_pretrained>
data = pd.read_csv('.. /input/train.csv')
 ID = data.pop('id')
 or_data = data.copy()
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.model_selection import GridSearchCV
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path )<load_pretrained>
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation
from keras.utils.np_utils import to_categorical
from keras.wrappers.scikit_learn import KerasClassifier
from keras.layers import BatchNormalization
model_path = f'.. /input/bert-base-tf2-0-minimalistic-iii/bert-1.h5'
model = bert_model()
model.load_weights(model_path )<concatenate>
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 y_cat = to_categorical(y)
 y_cat.shape, y.shape
models.append(model )<define_variables>
X = StandardScaler().fit(data ).transform(data)
 X.shape
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = StandardScaler().fit(data ).transform(test )
test_predictions = []<predict_on_test>
def create_model(dropout_rate_l1=0.3 , dropout_rate_l2=0.3):
model = Sequential()
model.add(Dense(600, input_dim=192, init='uniform'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(dropout_rate_l1))
model.add(Dense(300, init='uniform'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(dropout_rate_l2))
model.add(Dense(99, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 return model
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))<prepare_output>
X_train,X_test,y_train,y_test = train_test_split(X, y_cat, test_size = 0.1, random_state = 7 )
final_predictions = np.mean(test_predictions, axis=0 )<save_to_csv>
model = create_model()
history_main = model.fit(X_train,y_train,batch_size=192, nb_epoch=400, verbose=2, validation_data=(X_test, y_test))
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )<set_options>
model.evaluate(X, y_cat )
np.set_printoptions(suppress=True )<load_from_csv>
yPred = model.predict_proba(test)
 col = or_data.species.unique()
col.sort()
yPred = pd.DataFrame(yPred, index=index, columns=col)
 yPred.to_csv('result_2.csv', index=True )