Kaggle competition: Google QUEST Q&A Labeling
embedding_path = ".. /input/glove6b300dtxt/glove.6B.300d.txt"
def read_embedding_vec(embedding_file_path):
embedding_list = []
embedding_word_dict = dict()
f = open(embedding_file_path)
 for index, line in enumerate(f):
values = line.split()
word = values[0]
try:
coefs = np.asarray(values[1:], dtype='float32')
 except:
continue
embedding_list.append(coefs)
 embedding_word_dict[word] = len(embedding_word_dict)
 f.close()
embedding_list = np.array(embedding_list)
 return embedding_list, embedding_word_dict<split>
kf = np.load(".. /input/split-dataset/dataset_indexs.npy", allow_pickle=True )
embedding_list, embedding_word_dict = read_embedding_vec(embedding_path)
<concatenate>
train_df = pd.read_csv('.. /input/google-quest-challenge/train.csv')
 sub_df = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv' )
try:
embedding_word_dict['nanword']
except:
embedding_word_dict['nanword'] = len(embedding_word_dict)
 embedding_list = np.append(embedding_list, [np.zeros_like(embedding_list[0,:])], axis = 0)
 try:
embedding_word_dict['unknownword']
except:
embedding_word_dict['unknownword'] = len(embedding_word_dict)
 embedding_list = np.append(embedding_list, [np.zeros_like(embedding_list[0,:])], axis = 0)
<categorify>
model_names = ["albert", "bert_base", "tf_bert_large", "feat_multi", "feat_nn",
"tf_bert_base_epoch1", "tf_bert_base_epoch2", "tf_bert_base_epoch3"]
def preprocess(string):

 string = string.lower()
string = re.sub(r'(\")', '', string)
 string = re.sub(r'(\\r\ 
)', ' ', string)
 string = re.sub(r'(\\r)', ' ', string)
 string = re.sub(r'(\ 
)', ' ', string)
 string = re.sub(r'(\\)', ' ', string)
 string = re.sub(r'\\t', ' ', string)
 string = re.sub(r'\:', ' ', string)
 string = re.sub(r'<define_variables>
def compute_spearmanr(trues, preds):
rhos = []
for col_true, col_pred in zip(trues.T, preds.T):
rhos.append(spearmanr(col_true, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
words_dict = dict() 
<feature_engineering>
target_columns = list(sub_df.columns)
 results = []
for fold_i in range(1, 6):
result_df = pd.DataFrame(index=model_names, columns=target_columns[-30:] + ["ave"])
 for model_name in model_names:
for target in target_columns[-30:]:
true = train_df[target_columns].iloc[kf[fold_i-1][1]].reset_index(drop=True)
 oof = pd.read_csv(model_name + f"/oof{fold_i}.csv")
 r = spearmanr(true[target].values, oof[target].values + np.random.normal(0, 1e-7, oof.shape[0])).correlation
result_df.loc[model_name, target] = r
result_df.loc[model_name, "ave"] = np.mean(result_df.loc[model_name, target_columns[-30:]].values)
 results.append(result_df)
def combine_text(df):
try:
if len(df['text'])== len(df): return df
except: pass
df['text'] = ""
text_col = ['project_title', 'project_essay_1',
'project_essay_2', 'project_essay_3',
'project_essay_4', 'project_resource_summary']
df[text_col] = df[text_col].fillna('NANWORD')
 df['text'] = df.apply(lambda x: " ".join([str(x[col] ).strip() for col in text_col]), axis=1)
 return df<string_transform>
kfold = 5
model_number = len(model_names)
 true_dfs = []
for i in range(5):
oof = train_df.iloc[kf[i][1]]
true_dfs.append(oof[target_columns])
 model_df = defaultdict(lambda: [])
 for model_name in model_names:
print(model_name)
 scores = []
pred_dfs = []
for i in range(5):
pred = pd.read_csv(f"{model_name}/oof{i+1}.csv")
 model_df[model_name].append(pred)
 true = true_dfs[i]
rho = compute_spearmanr(true.values, pred.values)
 scores.append(rho)
 pred_dfs.append(pred.copy())
 print([f"fold{i+1}: {scores[i]:.4}" for i in range(5)])
 scores = [{} for _ in range(kfold)]
all_coeffs = []
for col_num, col_name in enumerate(target_columns[-30:]):
coefficients = np.zeros(( kfold, model_number))
print(col_name, "
")
 for fold_num,(_, vld_idx)in enumerate(kf):
print("
"+"-"*50+'
[Fold {}/{}]'.format(fold_num + 1, kfold))
def function_spearmanr(x):
true = true_dfs[fold_num].loc[:, col_name].values
pred = np.zeros(len(true))
for k,(model_name, df)in enumerate(model_df.items()):
pred += x[k] * df[fold_num].loc[:, col_name].values
return -spearmanr(true, pred ).correlation
x0 = [1 for _ in range(model_number)]
res = minimize(function_spearmanr, x0, method='nelder-mead', options={'xtol': 1e-4, 'disp': True})
 coeffs = [max(val, 0)for val in res.x]
coefficients[fold_num,:] = coeffs / np.sum(coeffs)
 print('
final vector: {}'.format(coefficients[fold_num,:]))
y_true = true_dfs[fold_num].loc[:, col_name].values
y_pred = np.zeros(len(y_true))
for k,(model_name, df)in enumerate(model_df.items()):
y_pred += coefficients[fold_num, k] * df[fold_num].loc[:, col_name].values
score = spearmanr(y_true, y_pred ).correlation
scores[fold_num][col_name] = score
print(f"valid's spearmanr: {score:<7.4f}
")
 all_coeffs.append(coefficients)
df_train = combine_text(df_train)
 df_train.head()<string_transform>
sub_df = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 submission = np.zeros(( len(sub_df), 30)).T
FOLD_NUM = 5
inf_sub = []
for fold_num in range(FOLD_NUM):
l = []
for model_num, model_name in enumerate(model_names):
l.append(pd.read_csv(f"{model_name}/submission{fold_num+1}.csv"))
inf_sub.append(l.copy() )
def tokenize_sentences(sentences, words_dict, early_stopping = False):

 tokenized_sentences = []
for sentence in tqdm.tqdm(sentences):
if hasattr(sentence, "decode"):
sentence = sentence.decode("utf-8")
 sentence = preprocess(sentence)
 tokens = nltk.tokenize.word_tokenize(sentence)
 result = []
for word in tokens:
word = word.lower()
if word not in words_dict:
words_dict[word] = len(words_dict)
 word_index = words_dict[word]
result.append(word_index)
 tokenized_sentences.append(result)
 if early_stopping:
print(sentence)
 if len(tokenized_sentences)== 50:
return tokenized_sentences, words_dict
return tokenized_sentences, words_dict<string_transform>
for col_num, col_name in enumerate(target_columns[-30:]):
for fold_num in range(FOLD_NUM):
for model_num in range(len(model_names)) :
coef = all_coeffs[col_num][fold_num][model_num]
submission[col_num] += coef / FOLD_NUM * inf_sub[fold_num][model_num][col_name].values
sub_df.iloc[:, -30:] = submission.T
sub_df
df_train['tokenized_text'], words_dict = tokenize_sentences(df_train['text'], words_dict )<feature_engineering>
def norm_sub(df):
for col_name in df.columns[-30:]:
tmp_df = df[col_name].values
v_max = np.max(tmp_df)+ 0.01
v_min = np.min(tmp_df)- 0.01
df[col_name] = df[col_name].apply(lambda x:(x - v_min)/(v_max - v_min))
df[col_name] = df[col_name].values + np.random.normal(0, 1e-7, len(df))
return df
sub_df = norm_sub(sub_df )
df_train["tokenized_text_length"] = df_train['tokenized_text'].apply(lambda x: len(x))<filter>
len(df_train[(df_train["project_is_approved"] == True)&(df_train["tokenized_text_length"]>= 630)] )<categorify>
class color:
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARKCYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
def compute_spearmanr(trues, preds, columns=None):
rhos = []
for i,(col_true, col_pred)in enumerate(zip(trues.T, preds.T)) :
if i == 0:
continue
r = spearmanr(col_true, col_pred ).correlation
if columns:
print("{:.3f}\t{}".format(r, columns[i-1]))
rhos.append(r)
 return np.mean(rhos)
 def compare_spearmanr(trues, preds, pp_preds, columns=None):
rhos = []
pp_rhos = []
for i,(col_true, col_pred, col_pp_pred)in enumerate(zip(trues.T, preds.T, pp_preds.T)) :
if i == 0:
continue
r = spearmanr(col_true, col_pred ).correlation
pp_r = spearmanr(col_true, col_pp_pred ).correlation
if columns:
if r < pp_r:
print("{:.3f} ->".format(r)+ color.BOLD + " {:.3f}".format(pp_r)+ color.END + " \t{}".format(columns[i-1]))
elif(r - pp_r)< 10**(-6):
print("{:.3f} ->".format(r)+ " {:.3f}".format(pp_r)+ " \t{}".format(columns[i-1]))
else:
print("{:.3f} ->".format(r)+ color.RED + " {:.3f}".format(pp_r)+ color.END + " \t{}".format(columns[i-1]))
rhos.append(r)
 pp_rhos.append(pp_r)
 whole_base = np.mean(rhos)
 whole_pp = np.mean(pp_rhos)
 print("Whole Score")
 if whole_base < whole_pp:
print("{:.3f} ->".format(whole_base)+ color.BOLD + " {:.3f}".format(whole_pp)+ color.END + " +{}".format(abs(whole_pp - whole_base)))
 elif(whole_base - whole_pp)< 10**(-6):
print("{:.3f} ->".format(whole_base)+ " {:.3f}".format(whole_pp))
else:
print("{:.3f} ->".format(whole_base)+ color.RED + " {:.3f}".format(whole_pp)+ color.END + " -{}".format(abs(whole_pp - whole_base)))
 return whole_base, whole_pp
def inf_numbert_of_annotators(v):
s = set(v)
 d = 10
for v in s:
for u in s:
if v != u:
d = min(abs(v - u), d)
 for i in range(100):
if d * i >= 1 - 10**(-3):
return i
def zero_one_scale(v):
max_x = np.max(v)
 min_x = np.min(v)
 return(v - min_x)/(max_x - min_x)
 def post_process(train_df, train, pred_df):
print(len(train), len(pred_df))
df = pred_df.copy()
df.loc[(train["host"] != "english.stackexchange.com")&(train["host"] != "ell.stackexchange.com"), "question_type_spelling"] = 0
for col in ["question_type_spelling", "question_type_entity", "question_type_definition",
"question_type_compare", "question_conversational", "question_interestingness_others",
"question_type_instructions", "question_type_consequence", "question_interestingness_self",
"question_has_commonly_accepted_answer", "answer_relevance", "question_not_really_a_question"]:
n = inf_numbert_of_annotators(train_df[col].values)
 df[col] = zero_one_scale(df[col].values)
 for i in range(n+1):
base = i / n
low = base - 1 /(2 * n)
 up = base + 1 /(2 * n)
 df.loc[(df[col] >= low)&(df[col] <= up), col] = base
if len(set(df[col].values)) == 1:
df[col] = pred_df[col].values
return df
def clear_embedding_list(embedding_list, embedding_word_dict, words_dict):

 cleared_embedding_list = []
cleared_embedding_word_dict = {}
for word in words_dict:
if word not in embedding_word_dict:
continue
word_id = embedding_word_dict[word]
row = embedding_list[word_id]
cleared_embedding_list.append(row)
 cleared_embedding_word_dict[word] = len(cleared_embedding_word_dict)
 return cleared_embedding_list, cleared_embedding_word_dict<categorify>
train_df = pd.read_csv('.. /input/google-quest-challenge/train.csv')
 test_df = pd.read_csv('.. /input/google-quest-challenge/test.csv')
def convert_tokens_to_embedded(tokenized_sentences, words_to_ids, embedding_word_dict,
sentences_length = 630, padding = embedding_word_dict['nanword'],
early_stopping = False):

 words_train = []
for sentence in tokenized_sentences:
current_words = []
for word_index in sentence:
word = words_to_ids[word_index]
word_id = embedding_word_dict.get(word, embedding_word_dict['unknownword'])
 current_words.append(word_id)
 if len(current_words)>= sentences_length:
current_words = current_words[:sentences_length]
else:
current_words += [padding] *(sentences_length - len(current_words))
if early_stopping and len(words_train)>= 10:
break
words_train.append(current_words)
 return words_train<define_variables>
sub_df.to_csv("submission.csv", index=False )