Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<save_to_csv>

 pyLDAvis.enable_notebook()
np.random.seed(2018)
 warnings.filterwarnings('ignore' )
model.load_weights(file_path)
 pred_test = model.predict([X_test_cat, X_test_num, X_test_words], batch_size=1024, verbose=1)
 test["project_is_approved"] = pred_test
if COMPLETE_RUN:
test[['id', 'project_is_approved']].to_csv("submission.csv", index=False )<save_to_csv>
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )
LABELS = ["project_is_approved"]
base = ".. /input/the-all-in-one-model-prediction-files/"
predict_list = []
for j in range(10):
predict_list.append(pd.read_csv(base + "submission_%d.csv"%j)[LABELS])
 predcitions = np.zeros_like(predict_list[0])
 for predict in predict_list:
predcitions = np.add(predcitions, logit(predict))
predcitions /= len(predict_list)
 predcitions = expit(predcitions)
 submission = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 submission[LABELS] = predcitions
submission.to_csv('submission.csv', index=False )<drop_column>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )
os.remove('best.h5' )<import_modules>
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )
start = time.time()
train = pd.read_csv(".. /input/train.csv",index_col="id",low_memory=False,
parse_dates=["project_submitted_datetime"])
 traindex = train.index
test = pd.read_csv(".. /input/test.csv",index_col="id",low_memory=False,
parse_dates=["project_submitted_datetime"])
 tesdex = test.index
y = train["project_is_approved"].copy()
df = pd.concat([train.drop("project_is_approved",axis=1),test],axis=0)
 alldex = df.index
rc = pd.read_csv(".. /input/resources.csv",index_col="id" ).fillna("missingpotato" )<merge>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
agg_rc = rc.reset_index().groupby('id' ).agg(
 dict(quantity = ['sum',"mean"],
price = ["sum","mean","max","min","std"],
id = 'count',
description = lambda x: ' nicapotato '.join(x)) ).rename(columns={"id" : "count"})
 agg_rc.columns = pd.Index([e[0] +"_"+ e[1] for e in agg_rc.columns.tolist() ])
 agg_rc.rename({'count_count':"count",'description_<lambda>': "description"}, axis=1,inplace=True)
 agg_rc.price_std.fillna(0,inplace=True)
 df = pd.merge(df,agg_rc, left_index=True, right_index=True, how= "left")
 del test, train, rc,agg_rc
gc.collect()<feature_engineering>
lengths = train['question_title'].apply(len)
 train['lengths'] = lengths
lengths = train.loc[train['lengths']<4000]['lengths']
sns.distplot(lengths, color='b')
 plt.show()
df['text'] = df.apply(lambda row: ' '.join([
str(row['project_essay_1']),
str(row['project_essay_2']),
str(row['project_essay_3']),
str(row['project_essay_4'])]), axis=1 )<feature_engineering>
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )
%%time
text_cols = ["text","project_resource_summary", "project_title", "description"]
print("Hand Made Text Features.. ")
 SIA = SentimentIntensityAnalyzer()
for cols in text_cols:
df[cols] = df[cols].astype(str)
 df[cols] = df[cols].str.lower()
df[cols+'_num_chars'] = df[cols].apply(len)
 df[cols+'_num_words'] = df[cols].apply(lambda comment: len(comment.split()))
df[cols+'_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 df[cols+'_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words']*100
if cols == "text":
df[cols+"_vader_Compound"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])
 df[cols+'_vader_Negative']= df[cols].apply(lambda x:SIA.polarity_scores(x)['neg'])
 df[cols+'_vader_Positive']= df[cols].apply(lambda x:SIA.polarity_scores(x)['pos'])
 print("{} Done".format(cols))<feature_engineering>
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value
df["Year"] = df["project_submitted_datetime"].dt.year
df["Date of Year"] = df['project_submitted_datetime'].dt.dayofyear
df["Weekday"] = df['project_submitted_datetime'].dt.weekday
df["Weekd of Year"] = df['project_submitted_datetime'].dt.week
df["Day of Month"] = df['project_submitted_datetime'].dt.day
df["Quarter"] = df['project_submitted_datetime'].dt.quarter
df = pd.merge(df, df["project_subject_categories"].str.get_dummies(sep=', '),
left_index=True, right_index=True, how="left")
 df = pd.merge(df, df["project_subject_subcategories"].str.get_dummies(sep=', '),
left_index=True, right_index=True, how="left")
 teachr_multi_subs = df['teacher_id'].value_counts().reset_index()
df["multi_apps"]= df['teacher_id'].isin(teachr_multi_subs.loc[teachr_multi_subs["teacher_id"]>1,'index'].tolist())
 print("Teacher App Distribution:
Two Apps: {}%
One App: {}%
".format(*df["multi_apps"].value_counts(normalize=True)*100))
df["Gender"] = None
df.loc[df['teacher_prefix'] == "Mr.","Gender"] = "Male"
df.loc[df['teacher_prefix'] == "Teacher","Gender"] = "Not Specified"
df.loc[(df['teacher_prefix'] == "Mrs.")|(df['teacher_prefix'] == "Ms."),"Gender"] = "Female"
print("Gender Distribution:
Female: {}%
Male: {}%
Not Specified: {}%".format(*df["Gender"].value_counts(normalize=True)*100))<categorify>
np.set_printoptions(suppress=True )
dumyvars= ["Gender",'school_state','project_grade_category']
timevars = ['Weekday','Weekd of Year','Day of Month','Year','Date of Year',"Quarter"]
encode = ['multi_apps', "Date_Cutoff", 'teacher_prefix',"teacher_id"]
categorical_features = dumyvars + timevars + encode
lbl = preprocessing.LabelEncoder()
for col in categorical_features:
df[col] = lbl.fit_transform(df[col].astype(str)) 
<drop_column>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
text_cols = ["project_resource_summary", "project_title","description","text"]
df.drop(['project_subject_categories',"project_subject_subcategories","project_submitted_datetime",
"project_essay_1","project_essay_2","project_essay_3","project_essay_4"
],axis=1,inplace=True)
 normalize = ["teacher_number_of_previously_posted_projects","quantity","price"]
gc.collect()<count_values>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
print("
Dtypes of DF features:
",df.dtypes.value_counts())
 print("
DF Shape: {} Rows, {} Columns".format(*df.shape))<define_search_space>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
tfidf_para = {
"sublinear_tf":True,
"strip_accents":'unicode',
"stop_words":"english",
"analyzer":'word',
"token_pattern":r'\w{1,}',
"dtype":np.float32,
"norm":'l2',
"min_df":5,
"max_df":.9,
"smooth_idf":False
}<feature_engineering>
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
def get_col(col_name):
return lambda x: x[col_name]
df["project_title_count"] = df["project_title"].copy()
textcols = ["text","project_resource_summary","project_title", "project_title_count","description"]
vectorizer = FeatureUnion([ 
('text',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=20000,
**tfidf_para,
preprocessor=get_col('text'))), 
('project_resource_summary',TfidfVectorizer(
 ngram_range=(1, 2),
**tfidf_para,
max_features=2000,
preprocessor=get_col('project_resource_summary'))), 
('project_title',TfidfVectorizer(
 ngram_range=(1, 2),
**tfidf_para,
max_features=1500,
preprocessor=get_col('project_title'))), 
('project_title_count',CountVectorizer(
 ngram_range=(1, 2),
max_features=1500,
preprocessor=get_col('project_title_count'))), 
('description',TfidfVectorizer(
 ngram_range=(1, 2),
**tfidf_para,
max_features=2400,
preprocessor=get_col('description'))),
] )<feature_engineering>
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path )
start_vect=time.time()
ready_df = vectorizer.fit_transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
print("Vectorization Runtime: %0.2f Minutes"%(( time.time() - start_vect)/60))<count_values>
model_path = f'.. /input/bert-base-tf2-0-minimalistic-iii/bert-1.h5'
model = bert_model()
model.load_weights(model_path )
df.drop(textcols,axis=1, inplace=True)
 print("
Dtypes of DF features:
",df.dtypes.value_counts())
 print("
DF Shape: {} Rows, {} Columns".format(*df.shape))<features_selection>
models.append(model )
def column_index(df, query_cols):
cols = df.columns.values
sidx = np.argsort(cols)
 return sidx[np.searchsorted(cols,query_cols,sorter=sidx)]
categorical_features_pos = column_index(df,categorical_features )<concatenate>
test_predictions = []
X = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]])
 testing = hstack([csr_matrix(df.loc[tesdex,:].values),ready_df[traindex.shape[0]:]])
 tfvocab = df.columns.tolist() + tfvocab
for shape in [X,testing]:
print("{} Rows and {} Cols".format(*shape.shape))
print("Feature Names Length: ",len(tfvocab))
del df
gc.collect() ;<count_values>
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))
imbalance_weight = y.value_counts(normalize=True)[0]/y.value_counts(normalize=True)[1]
print("Imbalance Weight: ",imbalance_weight )<train_model>
final_predictions = np.mean(test_predictions, axis=0 )
<init_hyperparams><EOS> 
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )