Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<prepare_x_and_y>
warnings.filterwarnings('ignore');
zipfile.ZipFile('/kaggle/input/ghouls-goblins-and-ghosts-boo/train.csv.zip' ).extractall()
zipfile.ZipFile('/kaggle/input/ghouls-goblins-and-ghosts-boo/test.csv.zip' ).extractall()
%matplotlib inline
class SpearmanRhoCallback(tf.keras.callbacks.Callback):
def __init__(self, training_data, validation_data, patience, model_name):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.patience = patience
self.value = -1
self.bad_epochs = 0
self.model_name = model_name
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_val = self.model.predict(self.x_val)
 rho_val = np.mean([spearmanr(self.y_val[:, ind],
y_pred_val[:, ind] ).correlation
for ind in range(y_pred_val.shape[1])])
 if rho_val >= self.value:
self.value = rho_val
self.bad_epochs = 0
self.model.save_weights(self.model_name)
 else:
self.bad_epochs += 1
if self.bad_epochs >= self.patience:
print("Epoch %05d: early stopping Threshold" % epoch)
 self.model.stop_training = True
print('\rval_spearman-rho: %s' %(str(round(rho_val, 4))), end=100 * ' ' + '
')
 logs['val_rho'] = rho_val
return rho_val
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
def spearman_loss(y_true, y_pred):
return K.var(y_pred - y_true, axis=-1)/(K.std(y_pred, axis=-1)*K.std(y_true, axis=-1))
def bert_model(bert_trainabel, learning_rate, len_tr, BATCH_SIZE, NUM_EPOCHS):
EMB_SIZE = 32
input_word_ids = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 input_category = tf.keras.layers.Input(( 1,), dtype=tf.int32, name='input_category')
 input_host = tf.keras.layers.Input(( 1,), dtype=tf.int32, name='input_host')
 category_emb = tf.keras.layers.SpatialDropout1D(0.1 )(
 tf.keras.layers.Embedding(input_dim=6, output_dim=EMB_SIZE )(input_category))
host_emb = tf.keras.layers.SpatialDropout1D(0.1 )(
 tf.keras.layers.Embedding(input_dim=65, output_dim=EMB_SIZE )(input_host))
features_dense = tf.keras.layers.concatenate([category_emb, host_emb], axis=1)
 features_dense = tf.keras.layers.Flatten()(features_dense)
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=bert_trainabel)
 pooled_output, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 print('pooled_output: ', pooled_output)
 print('sequence_output: ', sequence_output)
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.1 )(x)
 x = tf.keras.layers.concatenate([x, features_dense])
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(inputs=[input_word_ids, input_masks, input_segments, input_category, input_host],
outputs=out)
 decay_steps, warmup_steps = calc_train_steps(
 len_tr,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS, 
)
 adamW_opt = AdamWarmup(decay_steps=decay_steps, warmup_steps=warmup_steps, lr=learning_rate, min_lr=0,)
 model.compile(loss='binary_crossentropy', optimizer=adamW_opt)
 return model<load_pretrained>
def N_net(train, test, target):
hidden_layer_sizes=(100,)
 activation = 'relu'
solver = 'adam'
batch_size = 'auto'
alpha = 0.0001
random_state = 0
max_iter = 10000
early_stopping = True
clf = MLPClassifier(
 hidden_layer_sizes=hidden_layer_sizes,
activation=activation,
solver=solver,
batch_size=batch_size,
alpha=alpha,
random_state=random_state,
max_iter=max_iter, 
)
 clf.fit(train, target)
 SAVE_TRAINED_DATA_PATH = 'train1.learn'
joblib.dump(clf, SAVE_TRAINED_DATA_PATH)
 clf1 = joblib.load(SAVE_TRAINED_DATA_PATH)
 predict_no = clf1.predict(test)
 predict = np.round(np.round(predict_no, decimals=1))
return predict
MODEL_PATH = '.. /input/bert-weights-repeatkfold1224'
y_test = np.zeros(( len(test_inputs), len(targets)))
 all_predictions = []
start_time = time.time()
for fold_ in range(1, 11):
print('fold: ', fold_)
 K.clear_session()
model_name = f'{MODEL_PATH}/bert_fold{fold_}.h5'
model = bert_model(bert_trainabel=False, learning_rate=5e-5, len_tr=len(y_test), BATCH_SIZE=4, NUM_EPOCHS=4)
 model.load_weights(model_name)
 all_predictions.append(model.predict(test_inputs))
print("time elapsed: {:<5.2}m".format(( time.time() - start_time)/ 60))<prepare_output>
def N_netlog(train, test, target):
hidden_layer_sizes=(100,)
 activation = 'relu'
solver = 'adam'
batch_size = 'auto'
alpha = 0.0001
random_state = 0
max_iter = 10000
early_stopping = True
clf = MLPRegressor(
 hidden_layer_sizes=hidden_layer_sizes,
activation=activation,
solver=solver,
batch_size=batch_size,
alpha=alpha,
random_state=random_state,
max_iter=max_iter, 
)
 clf.fit(train, target)
 SAVE_TRAINED_DATA_PATH = 'train1.learn'
joblib.dump(clf, SAVE_TRAINED_DATA_PATH)
 clf1 = joblib.load(SAVE_TRAINED_DATA_PATH)
 predict = clf1.predict(test)
 return predict
test_preds2 = np.mean(all_predictions, axis=0 )<prepare_output>
def lightgbm(train, test, target):
X_train, X_test, Y_train, Y_test = train_test_split(train, target, random_state=0)
 params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_error',
'num_class': 3,
'verbose': -1
}
lgb_train = lgb.Dataset(X_train, Y_train)
 lgb_eval = lgb.Dataset(X_test, Y_test, reference=lgb_train)
 best_params, history = {}, []
gbm = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=100,
early_stopping_rounds=100 
)
 best_params = gbm.params
print("Best params:", best_params)
 params = best_params
predict_no = gbm.predict(test, num_iteration=gbm.best_iteration)
 predict = np.argmax(predict_no, axis=1)
 return predict
test_preds = test_preds1 * 0.5 + test_preds2 * 0.5<load_from_csv>
def SVM(train, test, target):
clf_result=svm.SVC(kernel='rbf', gamma=1/2 , C=1.0,class_weight='balanced', random_state=0)
 clf_result.fit(train, target)
 predict= np.array(clf_result.predict(test))
return predict
pred = np.copy(test_preds)
 hyper = pd.read_csv('.. /input/bert-base/hyper2.csv')
 submission = pd.read_csv(PATH+'sample_submission.csv')
 col1 = hyper.loc[hyper[['score1', 'score2']].max(axis=1)== hyper.score1, 'col'].tolist()
for col in col1:
colidx = target_col.index(col)
 pred[:, colidx] =(pred[:, colidx]//(1/90)) /90
for col in [x for x in target_col if x not in col1]:
if hyper.loc[hyper.col==col, 'pct'].values == 1:
pct = hyper.loc[hyper.col==col, 'pct'] - 0.005
elif hyper.loc[hyper.col==col, 'pct'].values == 0:
pct = hyper.loc[hyper.col==col, 'pct'] + 0.005
else:
pct = hyper.loc[hyper.col==col, 'pct']
changerow = int(len(df_test)* pct)
 colidx = target_col.index(col)
 if hyper.loc[hyper.col==col, 'choice'].values =='low':
rowidx = pred[:, colidx].argsort() [:changerow]
pred[rowidx, colidx] = 0
elif hyper.loc[hyper.col==col, 'choice'].values =='up':
rowidx = pred[:, colidx].argsort() [-changerow:]
pred[rowidx, colidx] = 1
else:
print('Wrong!')
 submission[target_col] = pred
submission.to_csv("submission.csv", index=False )<install_modules>
def LogRes(ghost,ghoul,goblin, test, target,HowDo):
vsnp= np.empty(( 529,6),dtype="float64")
 submission=np.empty(( 529),dtype="int")
 ghost0=np.zeros(len(ghost)) ;ghost1=np.ones(len(ghost))
ghoul0=np.zeros(len(ghoul)) ;ghoul1=np.ones(len(ghoul))
goblin0=np.zeros(len(goblin)) ;goblin1=np.ones(len(goblin))
vs1 = ghost.append(ghoul, ignore_index=True);vs1t = np.append(ghost0,ghoul1)
 vs2 = ghoul.append(goblin, ignore_index=True);vs2t = np.append(ghoul0,goblin1)
 vs3 = goblin.append(ghost, ignore_index=True);vs3t = np.append(goblin0,ghost1)
 if HowDo == "log":
model1 = LogisticRegression() ;model1.fit(vs1,vs1t);vsnp[:,0:2]=model1.predict_proba(test)
 model2 = LogisticRegression() ;model2.fit(vs2,vs2t);vsnp[:,2:4]=model2.predict_proba(test)
 model3 = LogisticRegression() ;model3.fit(vs3,vs3t);vsnp[:,4:6]=model3.predict_proba(test)
 if HowDo == "net":
vsnp[:,0]=1-N_netlog(vs1, test, vs1t);vsnp[:,1]=1-vsnp[:,0]
vsnp[:,2]=1-N_netlog(vs2, test, vs2t);vsnp[:,3]=1-vsnp[:,2]
vsnp[:,4]=1-N_netlog(vs3, test, vs3t);vsnp[:,5]=1-vsnp[:,4]
for n in range(len(vsnp)) :
if np.argmax(vsnp[n,:])==0:
submission[n]=0
if np.argmax(vsnp[n,:])==1:
submission[n]=2
if np.argmax(vsnp[n,:])==2:
submission[n]=2
if np.argmax(vsnp[n,:])==3:
submission[n]=1
if np.argmax(vsnp[n,:])==4:
submission[n]=1
if np.argmax(vsnp[n,:])==5:
submission[n]=0
print(vsnp)
 return submission
! pip install.. /input/sacremoses/sacremoses-master/
! pip install.. /input/transformers/transformers-master/<install_modules>
def LogRes2(ghost,ghoul,goblin, test, target,HowDo):
vsnp=np.empty(( 529,3),dtype="float64")
 submission=np.empty(( 529),dtype="int")
 ghost0=np.zeros(len(ghost)) ;ghost1=np.ones(len(ghost))
ghoul0=np.zeros(len(ghoul)) ;ghoul1=np.ones(len(ghoul))
goblin0=np.zeros(len(goblin)) ;goblin1=np.ones(len(goblin))
vs1 = ghost.append(ghoul, ignore_index=True)
 vs1 = vs1.append(goblin, ignore_index=True)
 vs1t = np.append(ghost0,ghoul1)
 vs1t = np.append(vs1t,goblin1)
 vs2 = ghoul.append(goblin, ignore_index=True)
 vs2 = vs2.append(ghost, ignore_index=True)
 vs2t = np.append(ghoul0,goblin1)
 vs2t = np.append(vs2t,ghost1)
 vs3 = goblin.append(ghost, ignore_index=True)
 vs3 = vs3.append(ghoul, ignore_index=True)
 vs3t = np.append(goblin0,ghost1)
 vs3t = np.append(vs3t,ghoul1)
 if HowDo == "log":
model1 = LogisticRegression() ;model1.fit(vs1,vs1t);vsnp[:,0]=model1.predict_proba(test)[:,0]
model2 = LogisticRegression() ;model2.fit(vs2,vs2t);vsnp[:,1]=model2.predict_proba(test)[:,0]
model3 = LogisticRegression() ;model3.fit(vs3,vs3t);vsnp[:,2]=model3.predict_proba(test)[:,0]
if HowDo == "net":
vsnp[:,0]=1-N_netlog(vs1, test, vs1t)
 vsnp[:,1]=1-N_netlog(vs2, test, vs2t)
 vsnp[:,2]=1-N_netlog(vs3, test, vs3t)
 for n in range(len(vsnp)) :
if np.argmax(vsnp[n,:])==0:
submission[n]=0
if np.argmax(vsnp[n,:])==1:
submission[n]=2
if np.argmax(vsnp[n,:])==2:
submission[n]=1
print(vsnp)
 return submission
!pip install.. /input/tensorflowv1/tensorboard-1.15.0-py3-none-any.whl
!pip install.. /input/tensorflowv1/tensorflow_estimator-1.15.1-py2.py3-none-any.whl
!pip install.. /input/tensorflowv1/tensorflow-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl 
<load_from_csv>
def syuunou(vote, ID):
pred:str=[]
for n in range(len(ID)) :
if np.argmax(vote[:,n])==0:
pred.append('Ghost')
 if np.argmax(vote[:,n])==2:
pred.append('Ghoul')
 if np.argmax(vote[:,n])==1:
pred.append('Goblin')
 s_c= pd.DataFrame({"id": ID, "type": pred})
 return s_c
PATH = '.. /input/google-quest-challenge/'
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories)
 MAX_LEN = 512
SEP_TOKEN_ID = 102
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
configPath = '.. /input/bert-model-config/'<set_options>
def tohyo(predict, vote):
vote[0] +=(predict==0);vote[1] +=(predict==1);vote[2] +=(predict==2 )
use_cuda = torch.cuda.is_available()
if use_cuda:
print('GPU available!')
 print(torch.cuda.get_device_name(0))
else:
print('GPU not available')
 device = torch.device("cuda:0" if use_cuda else "cpu")
<categorify>
def main_n() :
train = pd.read_csv('./train.csv')
 test = pd.read_csv('./test.csv')
 type_array = pd.get_dummies(train['type']); del train['type']
COLOR = pd.get_dummies(train['color']); del train['color'] ;del train['id']
COLOR2 = pd.get_dummies(test['color']); del test['color']; ID = test["id"]; del test['id']
vote = np.zeros(( 3,529),dtype = 'int')
 target = pd.DataFrame(type_array['Ghost']* 0 + type_array['Ghoul'] * 2 + type_array['Goblin'] * 1)
 ghost=train[type_array['Ghost']==1]
ghoul=train[type_array['Ghoul']==1]
goblin=train[type_array['Goblin']==1]
tohyo(N_net(train, test, target), vote)
 tohyo(lightgbm(train, test, target), vote)
 tohyo(SVM(train, test, target), vote)
 tohyo(LogRes2(ghost,ghoul,goblin, test, target,"log"), vote)
 tohyo(LogRes2(ghost,ghoul,goblin, test, target,"net"), vote)
 train = train.join(COLOR)
 test = test.join(COLOR2)
 ghost=train[type_array['Ghost']==1]
ghoul=train[type_array['Ghoul']==1]
goblin=train[type_array['Goblin']==1]
tohyo(LogRes2(ghost,ghoul,goblin, test, target,"net"), vote)
 tohyo(N_net(train, test, target), vote)
 submission = syuunou(vote, ID)
 return submission
def format_time(elapsed_time):

 elapsed_rounded = int(round(elapsed_time))
return str(datetime.timedelta(seconds=elapsed_rounded))<load_pretrained>
submission=main_n()
rows=submission
submission = rows
submission.to_csv("submission6.csv", index=False )
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, df, train_mode=True, labeled=True, target_columns=output_categories):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt', do_lower_case=True)
 self.target_columns = target_columns
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids = self.get_token_ids(row)
 if self.labeled:
labels = self.get_labels(row, self.target_columns)
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def get_token_ids(self, row):
title_tokens, ques_tokens, ans_tokens = self.trim_input(row.question_title, row.question_body, row.answer)
 tokens = ['[CLS]'] + title_tokens + ['[SEP]'] + ques_tokens + ['[SEP]'] + ans_tokens + ['[SEP]']
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if(len(token_ids)) < MAX_LEN:
token_ids += [0] *(MAX_LEN - len(token_ids))
token_ids_torch = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(token_ids_torch)
 return token_ids_torch, seg_ids
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_step = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == SEP_TOKEN_ID:
if first_step:
first_step = False
else:
seg_idx = 1
pad_idx = torch.nonzero(ids == 0)
 seg_ids[pad_idx] = 0
return seg_ids
def get_labels(self, row, target_columns):
return torch.tensor(row[target_columns].values.astype(np.float32))
def trim_input(self, title, question, answer, max_seq_length=MAX_LEN,
title_max_len=30, ques_max_len=239, ans_max_len=239,
head_len=120):
title_tokens = self.tokenizer.tokenize(title)
 ques_tokens = self.tokenizer.tokenize(question)
 ans_tokens = self.tokenizer.tokenize(answer)
 title_len = len(title_tokens)
 ques_len = len(ques_tokens)
 ans_len = len(ans_tokens)
 if(title_len + ques_len + ans_len + 4)> max_seq_length:
if title_max_len > title_len:
final_title_len = title_len
ans_max_len = int(ans_max_len + np.ceil(( title_max_len - title_len)/ 2))
ques_max_len = int(ques_max_len + np.floor(( title_max_len - title_len)/ 2))
else:
final_title_len = title_max_len
if ans_max_len > ans_len:
final_ans_len = ans_len
final_ques_len = ques_max_len +(ans_max_len - ans_len)
 elif ques_max_len > ques_len:
final_ques_len = ques_len
final_ans_len = ans_max_len +(ques_max_len - ques_len)
 else:
final_ans_len = ans_max_len
final_ques_len = ques_max_len
if final_ans_len + final_ques_len + final_title_len + 4 != max_seq_length:
print('Token Sequence Length:', final_ans_len + final_ques_len + final_title_len + 4)
 raise ValueError('New sequence length does not match.')
 title_tokens = title_tokens[:final_title_len]
if ques_len > final_ques_len:
max_head_len = max(head_len, int(0.5 * final_ques_len))
ques_tokens = ques_tokens[:max_head_len] + ques_tokens[max_head_len - final_ques_len:]
else:
ques_tokens = ques_tokens[:final_ques_len]
if ans_len > final_ans_len:
max_head_len = max(head_len, int(0.5 * final_ans_len))
ans_tokens = ans_tokens[:max_head_len] + ans_tokens[max_head_len - final_ans_len:]
else:
ans_tokens = ans_tokens[:final_ans_len]
return title_tokens, ques_tokens, ans_tokens<define_variables>
params = {'batch_size': 8,
'shuffle': False,
'num_workers': 6}<predict_on_test>