Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub['answer_relevance'] = df_sub['answer_relevance'].apply(lambda x : 0.33333334326744 if x < 0.7 else x)
 df_sub.to_csv('submission.csv', index=False )<define_variables>
%matplotlib inline
warnings.filterwarnings("ignore")
 plotly.offline.init_notebook_mode()
data_dir = '.. /input/'
NETLOC_PATH = '.. /input/bert-use-netloc/unique_netloc.pickle'
USE_URL = '.. /input/universalsentenceencoder4/'
TRAIN_CSV_PATH = path_join(data_dir,'google-quest-challenge', 'train.csv')
 TEST_CSV_PATH = path_join(data_dir, 'google-quest-challenge','test.csv')
 POST_PROCESS = False<set_options>
def flatten(d, parent_key='', sep='_'):

 items = []
for k, v in d.items() :
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(flatten(v, new_key, sep=sep ).items())
 else:
items.append(( new_key, v))
return dict(items)
 def showExecTime(startPoint, initialString = "", verbose=True):

 eex = time.time()
seconds = round(eex - startPoint, 2)
 minutes =(seconds/60)
 hours = int(minutes/60)
 minutes = int(minutes % 60)
 seconds = round(seconds % 60, 2)
 if verbose:
print("
- "+initialString+" Execution time: %sh %sm %ss -" %(hours, minutes, seconds))
return eex - startPoint
BertTokenizer, BertModel, BertForSequenceClassification, BertConfig,
WEIGHTS_NAME, CONFIG_NAME, AdamW, get_linear_schedule_with_warmup,
get_cosine_schedule_with_warmup, BertPreTrainedModel, 
)
 print(transformers.__version__)
 def set_seeds(SEED=42):
os.environ["PYTHONHASHSEED"] = str(SEED)
 np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
SEED=2019
set_seeds(SEED)
 HEAD_TAIL = True<load_from_csv>
test = pd.read_csv(".. /input/test.csv")
 train = pd.read_csv(".. /input/train.csv" )
test = pd.read_csv(TEST_CSV_PATH ).fillna(' ')
 train = pd.read_csv(TRAIN_CSV_PATH ).fillna(' ' )<string_transform>
colors = {
"Ghost" : "
"Ghoul" : "
"Goblin": "
}
train = clean_data(train, ['answer', 'question_body', 'question_title'])
 test = clean_data(test, ['answer', 'question_body', 'question_title'])
 tokenizer = Tokenizer()
full_text = list(train['question_body'])+ \
list(train['answer'])+ \
list(train['question_title'])+ \
list(test['question_body'])+ \
list(test['answer'])+ \
list(test['question_title'])
 tokenizer.fit_on_texts(full_text)
 embed_size=300
embedding_path = "/kaggle/input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl"
embedding_matrix, nb_words, unknown_words = build_matrix(embedding_path, path_join(data_dir,'wiki-news-300d-1M.vec'), tokenizer.word_index,
100000, embed_size )<set_options>
train['new_var'] = train['hair_length'] + train['has_soul'] + train['bone_length'] - train['rotting_flesh']
train['new_var'] = train['new_var'] - min(train['new_var'])
 train['new_var'] = train['new_var'] / max(train['new_var'])
 train.describe() ['new_var']
seed(SEED)
 random.seed(SEED )<load_from_csv>
train['bone_hair'] = train['hair_length'] + train['bone_length']
train['bone_hair'] = train['bone_hair'] - min(train['bone_hair'])
 train['bone_hair'] = train['bone_hair'] / max(train['bone_hair'])
 train['hair_soul'] = train['hair_length'] + train['has_soul']
train['hair_soul'] = train['hair_soul'] - min(train['hair_soul'])
 train['hair_soul'] = train['hair_soul'] / max(train['hair_soul'])
 train['bone_soul'] = train['bone_length'] + train['has_soul']
train['bone_soul'] = train['bone_soul'] - min(train['bone_soul'])
 train['bone_soul'] = train['bone_soul'] / max(train['bone_soul'])
 train['flesh_soul'] = train['rotting_flesh'] + train['has_soul']
train['flesh_soul'] = train['flesh_soul'] - min(train['flesh_soul'])
 train['flesh_soul'] = train['flesh_soul'] / max(train['flesh_soul'] )
train = pd.read_csv(TRAIN_CSV_PATH)
 test = pd.read_csv(TEST_CSV_PATH)
 print(train.shape, test.shape )<feature_engineering>
train['new_var_2'] =(train['hair_length'] * train['has_soul'] * train['bone_length'] *(1-train['rotting_flesh'])) **(1/4)
 train['bone_hair_2'] =(train['hair_length'] * train['bone_length'])**(1/2)
 train['flesh_soul_2'] =(train['rotting_flesh'] * train['has_soul'])**(1/2)
 train['hair_soul_2'] =(train['hair_length'] * train['has_soul'])**(1/2)
 train['bone_soul_2'] =(train['bone_length'] * train['has_soul'])**(1/2 )
find = re.compile(r"^[^.]*")
 train['netloc'] = train['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test['netloc'] = test['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 with open(NETLOC_PATH, "rb")as input_file:
unique_netloc = pickle.load(input_file)
 diff1 = list(set(test.netloc.unique() ).difference(set(unique_netloc)))
 unknownlist = ['unknown1','unknown2','unknown3', 'unknown4', 'unknown5', 'unknown6', 'unknown7', 'unknown8', 'unknown9',
'unknown10', 'unknown11', 'unknown12', 'unknown13', 'unknown14', 'unknown15', 'unknown16', 'unknown17', 'unknown18',
'unknown19', 'unknown20', 'unknown21', 'unknown22', 'unknown23', 'unknown24', 'unknown25', 'unknown26', 'unknown27',
'unknown28', 'unknown29', 'unknown30', 'unknown31', 'unknown32', 'unknown33', 'unknown34', 'unknown35', 'unknown36',
'unknown37', 'unknown38','unknown39', 'unknown40']
if len(diff1)> 50:
exit(1)
 for i,k in enumerate(diff1):
idx = test[test.netloc==k].index
test.iloc[idx,11] = unknownlist[i]
tmp = pd.DataFrame([['unknown1', 'LIFE_ARTS'],['unknown2', 'CULTURE'],['unknown3', 'SCIENCE'],
['unknown4', 'STACKOVERFLOW'],['unknown5', 'TECHNOLOGY'],['unknown6', 'LIFE_ARTS'],
['unknown7', 'CULTURE'],['unknown8', 'SCIENCE'],['unknown9', 'STACKOVERFLOW'],
['unknown10', 'TECHNOLOGY'],['unknown11', 'CULTURE'],['unknown12', 'SCIENCE'],
['unknown13', 'STACKOVERFLOW'],['unknown14', 'TECHNOLOGY'],['unknown15', 'LIFE_ARTS'],
['unknown16', 'LIFE_ARTS'],['unknown17', 'CULTURE'],['unknown18', 'SCIENCE'],
['unknown19', 'STACKOVERFLOW'],['unknown20', 'TECHNOLOGY'],['unknown21', 'LIFE_ARTS'],
['unknown22', 'CULTURE'],['unknown23', 'SCIENCE'],['unknown24', 'STACKOVERFLOW'],
['unknown25', 'TECHNOLOGY'],['unknown26', 'CULTURE'],['unknown27', 'SCIENCE'],
['unknown28', 'STACKOVERFLOW'],['unknown29', 'TECHNOLOGY'],['unknown30', 'LIFE_ARTS'] ,
['unknown31', 'LIFE_ARTS'],['unknown32', 'CULTURE'],['unknown33', 'SCIENCE'],
['unknown34', 'STACKOVERFLOW'],['unknown35', 'TECHNOLOGY'],['unknown36', 'LIFE_ARTS'],
['unknown37', 'CULTURE'],['unknown38', 'SCIENCE'],['unknown39', 'STACKOVERFLOW'],
['unknown40', 'TECHNOLOGY']
],columns = ['netloc', 'category'] )<categorify>
old_numerical = ['bone_length', 'rotting_flesh', 'hair_length', 'has_soul']
new_ones = ['new_var', 'bone_hair', 'flesh_soul', 'hair_soul', 'bone_soul']
new_ones_2 = [x+'_2' for x in new_ones]
new_ones, new_ones_2
features = ['netloc', 'category']
merged = pd.concat([train[features], test[features]])
 merged = merged.append(tmp[-(40 - len(diff1)) :])
 ohe = OneHotEncoder()
ohe.fit(merged)
 features_train = ohe.transform(train[features] ).toarray()
features_test = ohe.transform(test[features] ).toarray()
print(features_train.shape, features_test.shape)
<load_pretrained>
try:
train.drop(['flesh_soul', 'flesh_soul_2'], axis=1, inplace=True)
 new_ones.remove('flesh_soul')
 new_ones_2.remove('flesh_soul_2')
 except:
pass
test['new_var'] = test['hair_length'] + test['has_soul'] + test['bone_length'] - test['rotting_flesh']
test['new_var'] = test['new_var'] - min(test['new_var'])
 test['new_var'] = test['new_var'] / max(test['new_var'])
 test['bone_hair'] = test['hair_length'] + test['bone_length']
test['bone_hair'] = test['bone_hair'] - min(test['bone_hair'])
 test['bone_hair'] = test['bone_hair'] / max(test['bone_hair'])
 test['hair_soul'] = test['hair_length'] + test['has_soul']
test['hair_soul'] = test['hair_soul'] - min(test['hair_soul'])
 test['hair_soul'] = test['hair_soul'] / max(test['hair_soul'])
 test['bone_soul'] = test['bone_length'] + test['has_soul']
test['bone_soul'] = test['bone_soul'] - min(test['bone_soul'])
 test['bone_soul'] = test['bone_soul'] / max(test['bone_soul'])
 test['new_var_2'] =(test['hair_length'] * test['has_soul'] * test['bone_length'] *(1-test['rotting_flesh'])) **(1/4)
 test['bone_hair_2'] =(test['hair_length'] * test['bone_length'])**(1/2)
 test['hair_soul_2'] =(test['hair_length'] * test['has_soul'])**(1/2)
 test['bone_soul_2'] =(test['bone_length'] * test['has_soul'])**(1/2)
 color = pd.get_dummies(test['color'])
 test_data = pd.concat([test, color[all_colors]], axis = 1)
 color = pd.get_dummies(train['color'])
 train_data = pd.concat([train, color[all_colors]], axis = 1)
 train_data = train_data.drop('id', axis=1, inplace=False)
 test_data = test_data.drop('id', axis=1, inplace=False)
 train_data.columns - test_data.columns == ['type']
module_url = USE_URL
embed = hub.load(module_url )<categorify>
nb = {'name': 'Bernoulli NaiveBayes'}
nb['model'] = naive_bayes.BernoulliNB()
lr = {'name': 'Logistic Regression'}
lr['model'] = linear_model.LogisticRegression(solver='lbfgs')
 lrcv = {'name': 'Cross-Validated Logistic Regression'}
lrcv['model'] = linear_model.LogisticRegressionCV(Cs=100, solver='lbfgs', n_jobs=-1)
 svc = {'name': 'Support Vector Machine'}
svc['model'] = OneVsRestClassifier(svm.SVC(kernel='rbf', probability=True))
dtree = {'name': 'Decision Tree'}
dtree['model'] = tree.DecisionTreeClassifier(max_depth=20, min_samples_leaf=3)
 rf = {'name': 'Random Forest'}
rf['model'] = ensemble.RandomForestClassifier(n_estimators=1000, max_depth=20, min_samples_leaf=3, n_jobs=-2)
 k, w = 7, ['uniform', 'distance'][1]
knn = {'name': str(k)+'-Nearest Neighbors '+w}
knn['model'] = KNeighborsClassifier(n_neighbors=k, weights=w, algorithm='auto', n_jobs=-2)
 dummy_uni = {'name': 'Dummy MostFrequent'}
dummy_uni['model'] = dummy.DummyClassifier(strategy='most_frequent')
 dummy_str = {'name': 'Dummy Stratified'}
dummy_str['model'] = dummy.DummyClassifier(strategy='stratified' )
input_columns = ['question_title', 'question_body', 'answer']
embeddings_train = {}
embeddings_test = {}
for text in input_columns:
print(text)
 test_text = test[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_test_emb = []
batch_size = 4
ind = 0
ind = 0
while ind*batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
gc.collect() 
<set_options>
models = [nb, lr, lrcv, svc, dtree, rf, knn, dummy_uni, dummy_str]
torch.cuda.empty_cache()
gc.collect()<set_options>
def evaluate_models(models, features, verbose=True, n_splits=10):
for model in models:
clf = model['model']
if verbose: print("
"+model['name'])
 begin = time.time()
clf.fit(train_data[features], train_data['type'])
 model['fit_time'] = showExecTime(begin, model['name']+" fitted.", verbose)
 begin = time.time()
predicted_train = np.array(clf.predict_proba(train_data[features]))
model['prediction_time'] = showExecTime(begin, model['name']+" prediction complete.", verbose)
 logloss = log_loss(train_data['type'].values, predicted_train)
 model['train_log_loss'] = logloss
if verbose: print("
\tLog_loss on the train:", logloss)
 predicted_train = np.array(clf.predict(train_data[features]))
accuracy_score(train_data['type'], predicted_train)
 accuracy = accuracy_score(train_data['type'], predicted_train)
 model['train_accuracy'] = accuracy
if verbose: print("\tTrain accuracy:", accuracy)
 skf = model_selection.StratifiedKFold(n_splits=n_splits)
 logloss_train, logloss_test = [], []
accuracies = []
if verbose: print("
\tStratified K-Fold")
 for train_index, test_index in skf.split(train_data[features], train_data['type']):
x_train, x_test = train_data[features].iloc[train_index], train_data[features].iloc[test_index]
y_train, y_test = train_data['type'][train_index], train_data['type'][test_index]
clf.fit(x_train, y_train)
 predict_proba_train = np.array(clf.predict_proba(x_train))
predict_proba_test = np.array(clf.predict_proba(x_test))
logloss_train.append(log_loss(y_train, predict_proba_train))
logloss_test.append(log_loss(y_test, predict_proba_test))
predict_test = np.array(clf.predict(x_test))
accuracy = accuracy_score(y_test, predict_test)
 accuracies.append(accuracy)
 model['skf_validation'] = {
'logloss_train': {
'mean': np.mean(logloss_train),
'std': np.std(logloss_train)
 },
'logloss_test': {
'mean': np.mean(logloss_test),
'std': np.std(logloss_test)
 },
'accuracy': {
'mean': np.mean(accuracies),
'std': np.std(accuracies),
}
}
if verbose:
print("\tLog_loss on the train: %0.2f(+/- %0.2f)" %(np.mean(logloss_train), np.std(logloss_train)* 2))
print("\tLog_loss on the test: %0.2f(+/- %0.2f)" %(np.mean(logloss_test), np.std(logloss_test)* 2))
print("\tClassification accuracy: %0.2f(+/- %0.2f)" %(np.mean(accuracies), np.std(accuracies)* 2))
return models
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0 )<compute_test_metric>
all_features = all_colors + new_ones + new_ones_2 + old_numerical
print(all_features )
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 dist_features_test = np.array([
l2_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
cos_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding'])
 ] ).T
X_test = np.hstack([item for k, item in embeddings_test.items() ] + [features_test, dist_features_test])
<string_transform>
print("Validating group 1")
 models1 = {'data': evaluate_models(copy.deepcopy(models), all_colors, False),
'attr': 'only colors'}
print("Validating group 2")
 models2 = {'data': evaluate_models(copy.deepcopy(models), old_numerical, False),
'attr': 'old numerical'}
print("Validating group 3")
 models3 = {'data': evaluate_models(copy.deepcopy(models), old_numerical + new_ones + new_ones_2, False),
'attr': 'all numerical'}
print("Validating group 4")
 models4 = {'data': evaluate_models(copy.deepcopy(models), all_features, False),
'attr': 'all features'}
print("Validating group 5")
 models5 = {'data': evaluate_models(copy.deepcopy(models), new_ones, False),
'attr': 'new ones'}
print("Validating group 6")
 models6 = {'data': evaluate_models(copy.deepcopy(models), new_ones_2, False),
'attr': 'new ones squared'}
print("Validating group 7")
 models7 = {'data': evaluate_models(copy.deepcopy(models), new_ones + new_ones_2, False),
'attr': 'all new ones'}
max_len = 500
max_len_title = 30
train_question_tokenized = pad_sequences(tokenizer.texts_to_sequences(train['question_body']), maxlen = max_len)
 train_answer_tokenized = pad_sequences(tokenizer.texts_to_sequences(train['answer']), maxlen = max_len)
 train_title_tokenized = pad_sequences(tokenizer.texts_to_sequences(train['question_title']), maxlen = max_len_title)
 test_question_tokenized = pad_sequences(tokenizer.texts_to_sequences(test['question_body']), maxlen = max_len)
 test_answer_tokenized = pad_sequences(tokenizer.texts_to_sequences(test['answer']), maxlen = max_len)
 test_title_tokenized = pad_sequences(tokenizer.texts_to_sequences(test['question_title']), maxlen = max_len_title )<normalization>
model = lrcv['model']
features = new_ones_2
evaluate_models(copy.deepcopy([lrcv]), features)
 model.fit(train_data[features], train_data['type'])
 predicted_test = model.predict(test_data[features] )
<choose_model_class><EOS> 
submission = pd.concat([test['id'], pd.DataFrame(predicted_test)], axis=1)
 submission.columns = ['id', 'type']
submission.to_csv("new_sub.csv", header=True, index=False )