Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<compute_test_metric>

 pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )
def metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(loss_fn(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float)))
 return np.mean(metrics )<define_variables>
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )
N_STARTS = 14
S_STARTS = int(N_STARTS/2)
 res_relu = train_targets.copy()
res_elu = train_targets.copy()
res_relu.loc[:, train_targets.columns] = 0
res_elu.loc[:, train_targets.columns] = 0
ss_relu = sample_submission.copy()
ss_elu = sample_submission.copy()
ss_relu.loc[:, train_targets.columns] = 0
ss_elu.loc[:, train_targets.columns] = 0
ss_dict = {}
historys = dict()
tf.random.set_seed(42)
 for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=7, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f"======{train_targets.values[tr].shape}========{train_targets.values[te].shape}=====")
 if seed < S_STARTS:
print(f'Seed: {seed} => Fold: {n} ==>(RELU MODEL)')
 model = create_model(len(top_feats), actv='relu')
 else:
print(f'Seed: {seed} => Fold: {n} ==>(ELU MODEL)')
 model = create_model(len(top_feats), actv='elu')
 checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.2, min_lr=1e-6, patience=4, verbose=1, mode='auto')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'auto')
 early = EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience= 10, verbose = 1)
 history = model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=60, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt, early], verbose=2 
)
 historys[f'history_{seed+1}'] = history
print("Model History Saved.")
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 if seed < S_STARTS:
ss_relu.loc[:, train_targets.columns] += test_predict
res_relu.loc[te, train_targets.columns] += val_predict
else:
ss_elu.loc[:, train_targets.columns] += test_predict
res_elu.loc[te, train_targets.columns] += val_predict
print(f'OOF Metric For SEED {seed} => FOLD {n} : {metric(train_targets.loc[te, train_targets.columns], pd.DataFrame(val_predict, columns=train_targets.columns)) }')
 print('+-' * 10)
 ss_relu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_relu.loc[:, train_targets.columns] /= S_STARTS
ss_elu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_elu.loc[:, train_targets.columns] /= S_STARTS<compute_test_metric>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )
print(f'OOF Metric(relu): {metric(train_targets, res_relu)}')
 print(f'OOF Metric(elu): {metric(train_targets, res_elu)}' )<save_to_csv>
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )
ss_elu.to_csv('submission_elu.csv', index=False )<save_to_csv>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
ss_relu.to_csv('submission.csv', index=False )<set_options>
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )<define_variables>
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value
n_comp_GENES = 450
n_comp_CELLS = 2
VarianceThreshold_for_FS = 0.8
Dropout_Model = 0.25
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )<load_from_csv>
np.set_printoptions(suppress=True )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=1000,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
for i in range(7):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
test_predictions = []
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<filter>
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))
tail_cols = tail_cols.index<feature_engineering>
final_predictions = np.mean(test_predictions, axis=0 )
def get_tail_label(df):

 columns = df.columns
n = len(columns)
 irpl = np.zeros(n)
 for column in range(n):
irpl[column] = df[columns[column]].value_counts() [1]
irpl = max(irpl)/irpl
mir = np.average(irpl)
 tail_label = []
for i in range(n):
if irpl[i] > mir:
tail_label.append(columns[i])
 return tail_label
def get_index(df):

 tail_labels = get_tail_label(df)
 index = set()
for tail_label in tail_labels:
sub_index = set(df[df[tail_label]==1].index)
 index = index.union(sub_index)
 return list(index)
 def get_minority_instace(X, y):

 index = get_index(y)
 X_sub = X[X.index.isin(index)].reset_index(drop = True)
 y_sub = y[y.index.isin(index)].reset_index(drop = True)
 return X_sub, y_sub
def nearest_neighbour(X):

 nbs=NearestNeighbors(n_neighbors=120 ,metric='euclidean',algorithm='kd_tree' ).fit(X)
 euclidean,indices= nbs.kneighbors(X)
 return indices
def MLSMOTE(X,y, n_sample):

 indices2 = nearest_neighbour(X)
 n = len(indices2)
 new_X = np.zeros(( n_sample, X.shape[1]))
target = np.zeros(( n_sample, y.shape[1]))
for i in range(n_sample):
reference = random.randint(0,n-1)
 neighbour = random.choice(indices2[reference,1:])
 all_point = indices2[reference]
nn_df = y[y.index.isin(all_point)]
ser = nn_df.sum(axis = 0, skipna = True)
 target[i] = np.array([1 if val>2 else 0 for val in ser])
 ratio = random.random()
gap = X.loc[reference,:] - X.loc[neighbour,:]
new_X[i] = np.array(X.loc[reference,:] + ratio * gap)
 new_X = pd.DataFrame(new_X, columns=X.columns)
 target = pd.DataFrame(target, columns=y.columns)
 new_X = pd.concat([X, new_X], axis=0)
 target = pd.concat([y, target], axis=0)
 return new_X, target 
<prepare_x_and_y>
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
<prepare_x_and_y><EOS> 
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )