Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<load_from_csv>
import numpy as np
import pandas as pd
import seaborn as sns
import tensorflow as tf
import itertools
import os
import tflearn
from matplotlib import pyplot as plt
from sklearn.decomposition.kernel_pca import KernelPCA
from sklearn.metrics import classification_report
from sklearn.preprocessing import PolynomialFeatures
sub = pd.read_csv(".. /input/google-quest-qa-subm-files/final_submission.csv")
 sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv" )<concatenate>
%matplotlib inline
id_in_sub = set(sub.qa_id)
 id_in_sample_submission = set(sample_submission.qa_id)
 diff = id_in_sample_submission - id_in_sub
sample_submission = pd.concat([
sub,
sample_submission[sample_submission.qa_id.isin(diff)]
] ).reset_index(drop=True )<save_to_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission["type"] = "Unknown"
sample_submission.to_csv("submission.csv", index=False )<import_modules>
poly_features = PolynomialFeatures(interaction_only=True )
import numpy as np
import pandas as pd
import os
import sys 
<import_modules>
try_comb = pd.DataFrame(
 poly_features.fit_transform(train.drop(["id", "color", "type"], axis=1)) [:,5:],
columns=["boneXrotting", "boneXhair", "boneXsoul",
"rottingXhair", "rottingXsoul",
"hairXsoul"] 
)
 try_comb["type"] = train.type
import glob
from os.path import join as path_join
from scipy.stats import spearmanr, rankdata<install_modules>
for i in ["boneXhair", "boneXsoul", "hairXsoul"]:
train[i] = try_comb[i].copy()
try_comb = None
os.system('pip install.. /input/sacremoses/sacremoses-master/ > /dev/null')
 os.system('pip install.. /input/transformers/transformers-master/ > /dev/null' )<define_variables>
try_comb = pd.DataFrame(
 poly_features.fit_transform(test.drop(["id", "color"], axis=1)) [:,5:],
columns=["boneXrotting", "boneXhair", "boneXsoul",
"rottingXhair", "rottingXsoul",
"hairXsoul"] 
)
 for i in ["boneXhair", "boneXsoul", "hairXsoul"]:
test[i] = try_comb[i].copy()
try_comb = None
package_path = '.. /input/radam-pytorch/RAdam'
sys.path.append(package_path )<define_variables>
train.loc[train.type == "Ghost", "type"] = 0
train.loc[train.type == "Ghoul", "type"] = 1
train.loc[train.type == "Goblin", "type"] = 2
train["type"] = train["type"].astype("int" )
folder = '.. /input/google-quest-challenge/'
pretrained_bert = ".. /input/pretrainedbertpytorch/pretrained-bert-pytorch/bert-base-uncased/"
model_weight_path1 = '.. /input/quest-003-55-bert-07-3-training-1/'
model_weight_path2 = '.. /input/quest-003-55-bert-07-3-training-2/'
model_weight_path3 = '.. /input/quest-003-55-bert-07-3-training-3/'<set_options>
is_ghost =(train.type == 0 ).values
is_ghoul =(train.type == 1 ).values
is_goblin =(train.type == 2 ).values
warnings.simplefilter('ignore' )<import_modules>
ghost_ghoul = train.loc[is_ghost | is_ghoul].copy()
ghost_goblin = train.loc[is_ghost | is_goblin].copy()
ghoul_goblin = train.loc[is_ghoul| is_goblin].copy()
import torch
from torch.utils.data import TensorDataset, DataLoader, Dataset
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.utils.data.sampler import SubsetRandomSampler
from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR
from torch.nn.utils.weight_norm import weight_norm<normalization>
ghost_ghoul["KPCA_0"] = ghost_ghoul_transf[:,0]
ghost_ghoul["KPCA_1"] = ghost_ghoul_transf[:,1]
ghost_goblin["KPCA_0"] = ghost_goblin_transf[:,0]
ghost_goblin["KPCA_1"] = ghost_goblin_transf[:,1]
ghoul_goblin["KPCA_0"] = ghoul_goblin_transf[:,0]
ghoul_goblin["KPCA_1"] = ghoul_goblin_transf[:,1]
print("transformers:", transformers.__version__ )<define_variables>
def neural_net(X_train, y_train, X_test, layers=[1024], dropout=0.8, n_epoch=30):
if isinstance(X_train, pd.DataFrame):
X_train = X_train.values
if isinstance(y_train, pd.DataFrame):
y_train = y_train.values
if isinstance(X_test, pd.DataFrame):
X_test = X_test.values
with tf.Graph().as_default() :
net = tflearn.input_data(shape=[None, X_train.shape[1]])
 for layer_size in layers:
net = tflearn.fully_connected(net, layer_size,
activation='relu',
weights_init='xavier',
regularizer='L2')
 net = tflearn.dropout(net, dropout)
 net = tflearn.fully_connected(net, y_train.shape[1], activation='softmax')
 net = tflearn.regression(net)
 model = tflearn.DNN(net, tensorboard_verbose=0)
 model.fit(X_train, y_train, validation_set=0.2, n_epoch=n_epoch)
 probs = np.array(model.predict(X_test))
return probs
SEED = 12345<set_options>
X_train = ghost_ghoul.drop(["id", "color", "type"], axis=1)
 y_train = pd.get_dummies(ghost_ghoul["type"])
 X_test = test.drop(["id", "color"], axis=1)
 transf = ghost_ghoul_KPCA.transform(X_test)
 X_test["KPCA_0"] = transf[:,0]
X_test["KPCA_1"] = transf[:,1]
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )<load_pretrained>
ghost_ghoul_probs = neural_net(X_train, y_train, X_test, dropout=0.8, n_epoch=40 )
def read_pickle_from_file(pickle_file):
with open(pickle_file,'rb')as f:
x = pickle.load(f)
 return x
def write_pickle_to_file(pickle_file, x):
with open(pickle_file, 'wb')as f:
pickle.dump(x, f, pickle.HIGHEST_PROTOCOL )<load_from_csv>
X_train = ghost_goblin.drop(["id", "color", "type"], axis=1)
 y_train = pd.get_dummies(ghost_goblin["type"])
 X_test = test.drop(["id", "color"], axis=1)
 transf = ghost_goblin_KPCA.transform(X_test)
 X_test["KPCA_0"] = transf[:,0]
X_test["KPCA_1"] = transf[:,1]
train_df = pd.read_csv(f"{folder}train.csv")
 train_df.shape<load_from_csv>
ghost_goblin_probs = neural_net(X_train, y_train, X_test, layers=[512], dropout=0.8, n_epoch=60 )
test_df = pd.read_csv(f"{folder}test.csv")
 test_df.shape<define_variables>
X_train = ghoul_goblin.drop(["id", "color", "type"], axis=1)
 y_train = pd.get_dummies(ghoul_goblin["type"])
 X_test = test.drop(["id", "color"], axis=1)
 transf = ghoul_goblin_KPCA.transform(X_test)
 X_test["KPCA_0"] = transf[:,0]
X_test["KPCA_1"] = transf[:,1]
target_table = [
['question_asker_intent_understanding', 'question'],
['question_body_critical', 'question'],
['question_conversational', 'question'],
['question_expect_short_answer', 'question'],
['question_fact_seeking', 'question'],
['question_has_commonly_accepted_answer', 'question'],
['question_interestingness_others', 'question'],
['question_interestingness_self', 'question'],
['question_multi_intent', 'question'],
['question_not_really_a_question', 'question'],
['question_opinion_seeking', 'question'],
['question_type_choice', 'question'],
['question_type_compare', 'question'],
['question_type_consequence', 'question'],
['question_type_definition', 'question'],
['question_type_entity', 'question'],
['question_type_instructions', 'question'],
['question_type_procedure', 'question'],
['question_type_reason_explanation', 'question'],
['question_type_spelling', 'question'],
['question_well_written', 'question'],
['answer_helpful', 'answer'],
['answer_level_of_information', 'answer'],
['answer_plausible', 'answer'],
['answer_relevance', 'answer'],
['answer_satisfaction', 'answer'],
['answer_type_instructions', 'answer'],
['answer_type_procedure', 'answer'],
['answer_type_reason_explanation', 'answer'],
['answer_well_written', 'answer'],
]
input_columns = [
'question_title',
'question_body',
'answer'
]<define_variables>
ghoul_goblin_probs = neural_net(X_train, y_train, X_test, dropout=0.5, n_epoch=40 )
target_question_columns = []
target_answer_columns = []
for table in target_table:
if table[1] == 'question':
target_question_columns.append(table[0])
 elif table[1] == 'answer':
target_answer_columns.append(table[0])
 target_columns = target_question_columns + target_answer_columns
print('question:', len(target_question_columns))
print('answer:', len(target_answer_columns))
print('total:', len(target_columns))<feature_engineering>
global_predictions = np.zeros(( X_test.values.shape[0], 3))
global_predictions[:,[0,1]] += ghost_ghoul_probs
global_predictions[:,[0,2]] += ghost_goblin_probs
global_predictions[:,[1,2]] += ghoul_goblin_probs
for df in [train_df, test_df]:
df['domcom'] = df['url'].apply(lambda s: s.split('://')[1].split('/')[0].split('.'))
df['dom_cnt'] = df['domcom'].apply(lambda s: len(s))
df['domcom'] = df['domcom'].apply(lambda s: s + ['none', 'none'])
 for ii in range(0,4):
df['url_'+str(ii)] = df['domcom'].apply(lambda s: s[ii])
 df.drop('domcom', axis = 1, inplace = True )<create_dataframe>
test["global_pred"] = np.argmax(global_predictions, axis=1 ).astype("str")
 test.loc[test.global_pred == "0", "global_pred"] = "Ghost"
test.loc[test.global_pred == "1", "global_pred"] = "Ghoul"
test.loc[test.global_pred == "2", "global_pred"] = "Ghoblin"
train_feature = pd.DataFrame()
test_feature = pd.DataFrame()<string_transform>
submission["type"] = test["global_pred"]
submission.to_csv("sub.csv", index=False )