Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<compute_test_metric>
%matplotlib inline
print(check_output(["ls", ".. /input"] ).decode("utf8"))
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<split>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<load_pretrained>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
<load_pretrained>
le = LabelEncoder()
y = le.fit_transform(train.type.values)
 test_ids = test.id
train.drop(['type','id'],axis=1,inplace=True)
 test.drop(['id'],axis=1,inplace=True)
 all_data = pd.concat(( train,test),axis=0 ).reset_index()
all_data['hair_bone'] = all_data['hair_length'] * all_data['bone_length']
all_data['hair_rotting'] = all_data['hair_length'] * all_data['rotting_flesh']
all_data['hair_soul'] = all_data['hair_length'] * all_data['has_soul']
all_data['hair_soul_bone'] = all_data['hair_length'] * all_data['has_soul'] * all_data['bone_length']
all_data['hair_rotting_soul'] = all_data['hair_length'] * all_data['rotting_flesh'] * all_data['has_soul']
all_data['hair_bone_rotting'] = all_data['hair_length'] * all_data['bone_length'] *all_data['rotting_flesh']
all_data['rotting_hair_soul'] = all_data['rotting_flesh'] * all_data['hair_length'] * all_data['has_soul']
all_data['rotting_bone_soul'] = all_data['rotting_flesh'] * all_data['bone_length'] * all_data['has_soul']
all_data['bone_rotting_soul'] = all_data['bone_length'] * all_data['rotting_flesh'] * all_data['has_soul']
all_data['bone_rotting'] = all_data['bone_length'] * all_data['rotting_flesh']
all_data['bone_soul'] = all_data['bone_length'] * all_data['has_soul']
all_data['rotting_soul'] = all_data['rotting_flesh'] * all_data['has_soul']
all_data.drop(['index','color'],axis=1,inplace=True )
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
columns = all_data.columns
n_trains = train.shape[0]
scaler = StandardScaler().fit(all_data)
 x_train = scaler.transform(all_data.iloc[:n_trains])
 x_test = scaler.transform(all_data.iloc[n_trains:] )
for i in range(7):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
clf = RandomForestClassifier(n_estimators=300)
 clf.fit(x_train,y)
 indices = np.argsort(clf.feature_importances_)[::-1]
for idx in range(x_train.shape[1]):
print('%d feature %d %s(%f)' %(idx + 1,
indices[idx],
columns[indices[idx]],
clf.feature_importances_[indices[idx]]))
for i in range(7):
model_path = f'.. /input/bert-base-tf2-0-training/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<define_variables>
clf = RandomForestClassifier()
param = {'n_estimators':[150,250,350],
'criterion' : ['gini', 'entropy'],
'max_features' : ['auto', 'sqrt', 'log2', None]
}
optimzation = GridSearchCV(clf,param_grid = param , cv=5 , scoring='accuracy')
 optimzation.fit(x_train,y)
 print('Best score : ' , optimzation.best_score_)
 print('Best parameter : ' , optimzation.best_params_ )
test_predictions = []<predict_on_test>
clf = RandomForestClassifier(n_estimators=150,criterion='gini',max_features='log2')
 param = {'max_depth':[None,5,7,9,12],
'min_samples_split' : [3,5,7],
'min_weight_fraction_leaf' : [0.0,0.1],
'max_leaf_nodes' : [40,60,80,100]
}
optimzation = GridSearchCV(clf,param_grid = param , cv=5 , scoring='accuracy')
 optimzation.fit(x_train,y)
 print('Best score : ' , optimzation.best_score_)
 print('Best parameter : ' , optimzation.best_params_ )
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))<prepare_output>
clf = RandomForestClassifier(n_estimators=150,
criterion='gini',
max_features='log2',
min_samples_split=5,
min_weight_fraction_leaf=0.1,
max_leaf_nodes=100,
max_depth=None)
 clf.fit(x_train,y)
 y_pred = clf.predict(x_test)
 print(clf.classes_)
 print(le.classes_)
 pred = []
for yped in y_pred:
if yped ==0:
pred.append('Ghost')
 elif yped == 1:
pred.append('Ghoul')
 else:
pred.append('Goblin')
 result = pd.DataFrame({'id':test_ids,'type':pred})
 result.to_csv('randomforest_ghost.csv',index=False )
final_predictions = np.mean(test_predictions, axis=0 )<load_pretrained>
X_train,X_test,y_train,y_test = train_test_split(x_train,y,test_size=0.2,random_state=42)
 clf = SVC(kernel='rbf',probability=True)
 clf.fit(X_train,y_train)
 y_pred = clf.predict(X_test)
 print('Accuracy %f : ' %(accuracy_score(y_test,y_pred)) )
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.) 
<save_to_csv>
dtrain = xgb.DMatrix(x_train,label=y)
 dtest = xgb.DMatrix(x_test)
 xgb_params = {
'seed': 0,
'colsample_bytree': 0.7,
'silent': 1,
'subsample': 0.7,
'learning_rate': 0.075,
"objective": "multi:softmax",
'max_depth': 7,
'num_parallel_tree': 1,
'min_child_weight': 1,
'num_class':3
}
clf = xgb.train(xgb_params,dtrain,num_boost_round=350)
 y_pred = clf.predict(dtest)
 pred = []
for yped in y_pred:
if yped ==0:
pred.append('Ghost')
 elif yped == 1:
pred.append('Ghoul')
 else:
pred.append('Goblin')
 result = pd.DataFrame({'id':test_ids,'type':pred})
 result.to_csv('xgb_ghost.csv',index=False )