Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<save_to_csv>

 pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )
submission.to_csv('submission.csv', index=False )<define_variables>
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )
PATH = "data/dogscats/"<define_variables>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )
for r in random.sample(range(12499), 1000):
os.rename(f'{PATH}train/cats/cat.{r}.jpg', f'{PATH}valid/cats/cat.{r}.jpg')
 for r in random.sample(range(12499), 1000):
os.rename(f'{PATH}train/dogs/dog.{r}.jpg', f'{PATH}valid/dogs/dog.{r}.jpg' )<define_variables>
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )
sz=224
bs=20
arch=resnet50<load_pretrained>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)
 data = ImageClassifierData.from_paths(PATH, tfms=tfms, bs=bs,
trn_name='train',
test_name='test1',
val_name='valid')
 learn = ConvLearner.pretrained(arch, data, precompute=False )<train_model>
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )
learn.fit(0.01, 3, cycle_len=1 )<train_model>
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value
learn.unfreeze()
learn.bn_freeze(True)
 learn.fit(lrs=[1e-5,1e-4,1e-2], n_cycle=1,
cycle_len=1 )<feature_engineering>
np.set_printoptions(suppress=True )
log_preds,y = learn.TTA()<compute_test_metric>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
probs = np.mean(np.exp(log_preds),0)
 accuracy_np(probs, y), metrics.log_loss(y, probs )<predict_on_test>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
prediction = learn.predict(is_test=True)
 pred_test = np.argmax(prediction, axis=1)
 label_probs = np.exp(prediction[:,1] )<save_to_csv>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'arj_bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=epochs, batch_size=batch_size)
 model.save_weights(f'arj_bert-{fold}.h5')
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val)
submission = pd.DataFrame({'id':os.listdir(f'{PATH}test1'), 'label':label_probs})
 submission['id'] = submission['id'].map(lambda x: x.split('.')[0])
 submission['id'] = submission['id'].astype(int)
 submission = submission.sort_values('id')
 submission.to_csv('.. /working/submission.csv', index=False )<load_from_zip>
test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
! rm -rf data<load_from_csv>
models = []
for i in range(1,4):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
%%time
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )<drop_column>
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<categorify>
for i in range(3,5):
model_path = f'.. /input/using-pretrained-kernel/using_pretrained_kernel/arj_bert-{0}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
%%time
traintest = pd.concat([train, test])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:train.shape[0], :]
test_ohe = dummies.iloc[train.shape[0]:, :]
print(train_ohe.shape)
 print(test_ohe.shape )<split>
for i in range(3-5):
model_path = f'.. /input/bert-pretrained-models/Pretrained_bert_models/bert-{0}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
%%time
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = KFold(n_splits=5)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0]))
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 print(label + ' cv score {}: {}'.format(i, cv_score))
i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores}
return results
def runLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2
lr_params = {'solver': 'lbfgs', 'C': 0.1}
results = run_cv_model(train_ohe, test_ohe, target, runLR, lr_params, auc, 'lr' )<save_to_csv>
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
%%time
submission = pd.DataFrame({'id': test_id, 'target': results['test']})
 submission.to_csv('submission.csv', index=False )<install_modules>
for i in range(2,5):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
!pip install target_encoding 
<import_modules>
for i in range(3,6):
model_path = f'.. /input/bert-base-tf2-0-training/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
import pandas as pd
from target_encoding import TargetEncoderClassifier, TargetEncoder
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression, LogisticRegression<load_from_csv>
test_predictions = []
train=pd.read_csv(".. /input/cat-in-the-dat/train.csv")
 test=pd.read_csv(".. /input/cat-in-the-dat/test.csv")
 sample_submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv')
 cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42 )<categorify>
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))
len_uniques = []
for c in train.columns.drop(['id', 'target']):
le = LabelEncoder()
le.fit(pd.concat([train[c], test[c]]))
train[c] = le.transform(train[c])
 test[c] = le.transform(test[c])
 print(c, len(le.classes_))
len_uniques.append(len(le.classes_))
X = train.drop(['target', 'id'], axis=1)
 y = train['target']<define_variables>
final_predictions = np.mean(test_predictions, axis=0 )
ALPHA = 75
MAX_UNIQUE = max(len_uniques)
 FEATURES_COUNT = X.shape[1]<compute_train_metric>
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
<save_to_csv><EOS> 
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )