Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<load_pretrained>
USE_SAMPLE = False
USE_ONLY_SELECTED_FOLDS = True
selected_folds = [0,1,2]
WEIGHTS = 'softmax'
ROUND = True
USE_LGB = True
USE_XGB = False
USE_3SPLIT_BERT = False
USE_TEST_SAFETY_ADJUSTMENTS = True
beta = 18.5
opts = {}
opts["D"] = 2 ** 25
opts["learning_rate"] = 0.1
opts["n_passes"] = 80
opts["errors_satisfied"] = 0
opts["random_init"] = False
opts["clean"] = True
opts["2grams"] = True
%time
weights = train_tron(PATH+"labeledTrainData.tsv",opts )<split>
!pip install.. /input/sacremoses0038 > /dev/null
sys.path.insert(0, ".. /input/tokenizers0011/" )
%time
preds = test_tron(PATH+"testData.tsv",weights,opts )<save_to_csv>
!pip install.. /input/transformers241 > /dev/null --no-dependencies
with open("submit_perceptron.csv","wb")as outfile:
outfile.write('"id","sentiment"
'.encode('utf-8'))
for p in sorted(preds):
outfile.write("{},{}
".format(p[0],p[3] ).encode('utf-8'))<import_modules>
!pip install.. /input/fastparquet/fastparquet-0.3.2-cp36-cp36m-linux_x86_64.whl > /dev/null --no-dependencies
import json
import re
import nltk
from bs4 import BeautifulSoup
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from tensorflow.python.keras.preprocessing.text import Tokenizer
import zipfile<set_options>
!pip install.. /input/fastparquet/thrift-0.13.0-cp36-cp36m-linux_x86_64.whl > /dev/null --no-dependencies
%matplotlib inline<load_pretrained>
np.set_printoptions(suppress=True)
 print(tf.__version__)
 pd.set_option('display.max_colwidth', 500)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 COLOR = 'black'
matplotlib.rcParams['text.color'] = COLOR
matplotlib.rcParams['axes.labelcolor'] = COLOR
matplotlib.rcParams['xtick.color'] = COLOR
matplotlib.rcParams['ytick.color'] = COLOR
pd.set_option('display.max_colwidth', 500)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500 )
nltk.download('stopwords')
<load_from_csv>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
GPT2_PATH = '.. /input/gpt2-hugginface-pretrained/'
XLNET_PATH = '.. /input/xlnet-huggingface-pretrained/'
MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories)
 if USE_SAMPLE:
df_train = df_train.iloc[0:round(0.05*df_train.shape[0]),:]
df_test = df_test.iloc[0:round(0.1*df_test.shape[0]),:]
raw_train_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip", delimiter='\t')
 raw_train_data.head()<feature_engineering>
def save_file(var, name):
pickle.dump(var, open(f"/kaggle/working/{name}.p", "wb"))
raw_train_len = raw_train_data["review"].apply(len)
 raw_train_len<count_values>
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id if USING_PAD_TOKEN else 0
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(title, question, 'longest_first', max_sequence_length)
 input_ids_qa, input_masks_qa, input_segments_qa = return_id(title + ' ' + question, answer, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_qa, input_masks_qa, input_segments_qa,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_qa, input_masks_qa, input_segments_qa = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_qa, masks_qa, segments_qa, ids_a, masks_a, segments_a = _convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_qa.append(ids_qa)
 input_masks_qa.append(masks_qa)
 input_segments_qa.append(segments_qa)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_qa, dtype=np.int32),
np.asarray(input_masks_qa, dtype=np.int32),
np.asarray(input_segments_qa, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
print('number of + : {}'.format(raw_train_data['sentiment'].value_counts() [1]))
print('number of - : {}'.format(raw_train_data['sentiment'].value_counts() [0]))<string_transform>
xlnetcfg = {'architectures': ['XLNetLMHeadModel'],
'attn_type': 'bi',
'bi_data': False,
'bos_token_id': 0,
'clamp_len': -1,
'd_head': 64,
'd_inner': 3072,
'd_model': 768,
'do_sample': False,
'dropout': 0.1,
'end_n_top': 5,
'eos_token_ids': 0,
'ff_activation': 'gelu',
'finetuning_task': None,
'id2label': {0: 'LABEL_0', 1: 'LABEL_1'},
'initializer_range': 0.02,
'is_decoder': False,
'label2id': {'LABEL_0': 0, 'LABEL_1': 1},
'layer_norm_eps': 1e-12,
'length_penalty': 1.0,
'max_length': 20,
'mem_len': None,
'model_type': 'xlnet',
'n_head': 12,
'n_layer': 12,
'num_beams': 1,
'num_labels': 2,
'num_return_sequences': 1,
'output_attentions': False,
'output_hidden_states': False,
'output_past': True,
'pad_token_id': 0,
'pruned_heads': {},
'repetition_penalty': 1.0,
'reuse_len': None,
'same_length': False,
'start_n_top': 5,
'summary_activation': 'tanh',
'summary_last_dropout': 0.1,
'summary_type': 'last',
'summary_use_proj': True,
'temperature': 1.0,
'top_k': 50,
'top_p': 1.0,
'torchscript': False,
'untie_r': True,
'use_bfloat16': False,
'vocab_size': 32000}
train_word_cnt = raw_train_data['review'].apply(lambda x:len(x.split(' ')))
 train_word_cnt<feature_engineering>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def create_nn_model(output_len, model_type):
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 if model_type == 'BERT':
config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
elif model_type == 'GPT2':
config = GPT2Config()
gpt2_model = TFGPT2Model.from_pretrained(GPT2_PATH+'gpt2-tf_model.h5', config=config)
 q_embedding = gpt2_model(q_id)[0]
elif model_type == 'XLNET':
config = XLNetConfig.from_dict(xlnetcfg)
 xlnet_model = TFXLNetModel.from_pretrained(XLNET_PATH+'xlnet-tf_model.h5', config=config)
 q_embedding = xlnet_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
x = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 x = tf.keras.layers.Dense(output_len, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn], outputs=x)
 return model
r_qmarks = np.mean(raw_train_data['review'].apply(lambda x: '?' in x))
r_fullstops = np.mean(raw_train_data['review'].apply(lambda x: '.' in x))
r_capitals = np.mean(raw_train_data['review'].apply(lambda x: max(y.isupper() for y in x)))
 r_numbers = np.mean(raw_train_data['review'].apply(lambda x: max(y.isdigit() for y in x)))
 print('물음표가 있는 리뷰 : {:.2f}%'.format(r_qmarks * 100))
print('마침표가 있는 리뷰 : {:.2f}%'.format(r_fullstops * 100))
print('대문자가 있는 리뷰 : {:.2f}%'.format(r_capitals * 100))
print('숫자가 있는 리뷰 : {:.2f}%'.format(r_numbers * 100))<categorify>
question_only = ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written']
review = raw_train_data['review'][0]
review_text = BeautifulSoup(review,"html5lib" ).get_text()
review_text = re.sub("[^a-zA-Z]"," ",review_text )<string_transform>
answer_and_question = ['answer_level_of_information', 'answer_helpful','answer_plausible','answer_relevance','answer_satisfaction']
stop_words = set(stopwords.words('english'))
review_text = review_text.lower()
words = review_text.split()
words = [w for w in words if not w in stop_words]<string_transform>
answer_only = ['answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']
clean_review = ' '.join(words)
 print(clean_review )<string_transform>
AQ_and_AO = answer_and_question + answer_only
def preprocess(review,remove_stopwords = False):
review_text = BeautifulSoup(review,"html5lib" ).get_text()
review_text = re.sub("[^a-zA-Z]"," ",review_text)
 words = review_text.lower().split()
if remove_stopwords:
stop_words = set(stopwords.words('english'))
words = [w for w in words if not w in stop_words]
clean_review = ' '.join(words)
 return clean_review<drop_column>
gkf10 = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 gkf5 = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 common_validation_idx = []
val10 = []
val5 = []
val10_fold0 = None
for fold,(train_idx, valid_idx)in enumerate(gkf10):
if fold in selected_folds:
val10 += list(valid_idx)
 if fold == 0:
val10_fold0 = valid_idx
for fold,(train_idx, valid_idx)in enumerate(gkf5):
if fold in selected_folds:
val5 += list(valid_idx)
 common_validation_idx = np.array(list(set(val5 ).intersection(set(val10))))
clean_train_reviews = []
for review in raw_train_data['review']:
clean_train_reviews.append(preprocess(review,remove_stopwords = True))<create_dataframe>
def predict_nn(train_data, valid_data, test_data, weights, model_type):
K.clear_session()
model = create_nn_model(train_data[1].shape[1], model_type)
 model.load_weights(weights)
 trn_preds = np.zeros(train_data[1].shape)
 val_preds = model.predict(valid_data[0])
 print(f'Lengths of test list is {len(test_data)}')
 test_preds = model.predict(test_data)if test_data is not None else None
rho_trn = compute_spearmanr_ignore_nan(train_data[1], trn_preds)
 rho_val = compute_spearmanr_ignore_nan(valid_data[1], val_preds)
 print(f'Score train {rho_trn}; Score validation {rho_val}')
 return trn_preds, val_preds, test_preds
def get_cross_fold_preds_nn(input_idx, model_type):
all_fold_trn_preds, all_fold_val_preds, all_fold_test_preds = [],[],[]
all_fold_trn_outputs, all_fold_val_outputs = [], []
gkf = GroupKFold(n_splits=n_splits ).split(X=df_train.question_body, groups=df_train.question_body)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
if(USE_ONLY_SELECTED_FOLDS &(fold in selected_folds)) or(not USE_ONLY_SELECTED_FOLDS):
if MOD_DATA_STUCTURE == '2 split':
if input_idx == [0,1,2]: output_idx = [i for i, z in enumerate(output_categories)if z in question_only]
if input_idx == [3,4,5]: output_idx = [i for i, z in enumerate(output_categories)if z in AQ_and_AO]
if MOD_DATA_STUCTURE == '3 split':
if input_idx == [0,1,2]: output_idx = [i for i, z in enumerate(output_categories)if z in question_only]
if input_idx == [3,4,5]: output_idx = [i for i, z in enumerate(output_categories)if z in answer_and_question]
if input_idx == [6,7,8]: output_idx = [i for i, z in enumerate(output_categories)if z in answer_only]
train_inputs = [inputs[i][train_idx] for i in input_idx]
train_outputs = outputs[np.array(train_idx)[:,None], output_idx]
all_fold_trn_outputs.append(train_outputs)
 valid_inputs = [inputs[i][valid_idx] for i in input_idx]
valid_outputs = outputs[np.array(valid_idx)[:,None], output_idx]
all_fold_val_outputs.append(valid_outputs)
 current_test_input = [test_inputs[i] for i in input_idx]
print(f'Fold {fold}')
 input_type = None
if(input_idx == [3,4,5])&(model_type == 'XLNET'):
weights_path = f".. /input/gq-xlnet-pretrained/XLNET_question_only_fold_{fold}.h5"
input_type = 'question and answer'
elif(input_idx == [0,1,2])&(model_type == 'XLNET'):
weights_path = f".. /input/gq-xlnet-pretrained/XLNET_question_answer_fold_{fold}.h5"
input_type = 'question only'
if(input_idx == [3,4,5])&(model_type != 'XLNET'):
print(f'Using weights for BERT fold {fold}, question and answer modification')
 weights_path = f".. /input/{model_roor_dir}/{model_type}_question_answer_fold_{fold}.h5"
input_type = 'question and answer'
elif(input_idx == [0,1,2])&(model_type != 'XLNET'):
weights_path = f".. /input/{model_roor_dir}/{model_type}_question_only_fold_{fold}.h5"
input_type = 'question only'
elif(input_idx == [6,7,8])&(model_type != 'XLNET'):
weights_path = f".. /input/{model_roor_dir}/{model_type}_answer_only_fold_{fold}.h5"
input_type = 'answer only'
trn_preds, val_preds, test_preds = predict_nn(( train_inputs, train_outputs),(valid_inputs, valid_outputs), current_test_input, weights_path, model_type)
 all_fold_trn_preds.append(trn_preds)
 all_fold_val_preds.append(val_preds)
 all_fold_test_preds.append(test_preds)
 trn_preds, val_preds = np.concatenate(all_fold_trn_preds), np.concatenate(all_fold_val_preds)
 trn_out, val_out = np.concatenate(all_fold_trn_outputs), np.concatenate(all_fold_val_outputs)
 test_preds = np.stack(all_fold_test_preds, axis=2)
 test_preds = np.mean(test_preds, axis=2)
 print(f'Finished all folds for {model_type} {input_type}')
 print(test_preds.shape, trn_out.shape, val_preds.shape, val_out.shape, test_preds.shape)
 return(trn_preds, trn_out),(val_preds, val_out), test_preds
def get_nn_all_outputs(model_type):
print('Getting all folds for QUESTION ONLY')
 qonly_trn, qonly_val, qonly_tst = get_cross_fold_preds_nn([0,1,2], model_type)
 print('Getting all folds for QUESTION ANSWER')
 qa_trn, qa_val, qa_tst = get_cross_fold_preds_nn([3,4,5], model_type)
 if MOD_DATA_STUCTURE == '3 split':
print('Getting all folds for ANSWER ONLY')
 ao_trn, ao_val, ao_tst = get_cross_fold_preds_nn([6,7,8], model_type)
 trn =(np.concatenate(( qonly_trn[0], qa_trn[0], ao_trn[0]), axis=1), np.concatenate(( qonly_trn[1], qa_trn[1], ao_trn[1]), axis=1))
val =(np.concatenate(( qonly_val[0], qa_val[0], ao_val[0]), axis=1), np.concatenate(( qonly_val[1], qa_val[1], ao_val[1]), axis=1))
tst = np.concatenate(( qonly_tst, qa_tst, ao_tst), axis=1)
 if MOD_DATA_STUCTURE == '2 split':
trn =(np.concatenate(( qonly_trn[0], qa_trn[0]), axis=1), np.concatenate(( qonly_trn[1], qa_trn[1]), axis=1))
val =(np.concatenate(( qonly_val[0], qa_val[0]), axis=1), np.concatenate(( qonly_val[1], qa_val[1]), axis=1))
tst = np.concatenate(( qonly_tst, qa_tst), axis=1)
 print(f'Finsihed entire dataset(qonly and qa)for {model_type}')
 print(trn[0].shape, trn[1].shape, val[0].shape, val[1].shape, tst.shape)
 save_file(trn, f'{model_roor_dir}_trn')
 save_file(val, f'{model_roor_dir}_val')
 save_file(tst, f'{model_roor_dir}_tst')
 return trn, val, tst
clean_train_df = pd.DataFrame({'id':raw_train_data['id'], 'review':clean_train_reviews, 'sentiment':raw_train_data['sentiment']} )<string_transform>
tokenizer = XLNetTokenizer.from_pretrained('.. /input/gq-manual-uploads/xlnet tokenizer from colab/')
 USING_PAD_TOKEN = False
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 save_file(outputs, 'XLNET_outputs')
 save_file(inputs, 'XLNET_inputs')
 save_file(test_inputs, 'XLNET_test_inputs' )
tokenizer = Tokenizer(oov_token='<UNK>')
 tokenizer.fit_on_texts(clean_train_reviews)
 text_sequences = tokenizer.texts_to_sequences(clean_train_reviews )<string_transform>
model_roor_dir = 'gq-xlnet-pretrained'
MOD_DATA_STUCTURE = '2 split'
n_splits = 10
xlnet_trn, xlnet_val, xlnet_tst = get_nn_all_outputs('XLNET' )
word_vocab = tokenizer.word_index
print(word_vocab)
<define_variables>
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 USING_PAD_TOKEN = True
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 save_file(outputs, 'BERT_outputs')
 save_file(inputs, 'BERT_inputs')
 save_file(test_inputs, 'BERT_test_inputs' )
data_configs = {}
data_configs['vocab'] = word_vocab
data_configs['vocab_size'] = len(word_vocab)+ 1<train_model>
model_roor_dir = 'gq-bert-pretrained'
MOD_DATA_STUCTURE = '2 split'
n_splits = 10
bert_trn, bert_val, bert_tst = get_nn_all_outputs('BERT' )
MAX_SEQUENCE_LENGTH = 174
train_inputs = pad_sequences(text_sequences,maxlen=MAX_SEQUENCE_LENGTH, padding='post')
 print('shape of train data: ', train_inputs.shape )<define_variables>
model_roor_dir = '3rd-training-2nd-gen-bert-download-from-gdrive'
MOD_DATA_STUCTURE = '3 split'
n_splits = 10
selected_folds = [0]
bert_trn_3split, bert_val_3split, bert_tst_3split = get_nn_all_outputs('BERT')
 selected_folds = [0,1,2]
DATA_IN_PATH = './data_in/'
TRAIN_INPUT_DATA = 'train_input.npy'
TRAIN_LABEL_DATA = 'train_label.npy'
TRAIN_CLEAN_DATA = 'train_clean.csv'
DATA_CONFIGS = 'data_configs.json'
if not os.path.exists(DATA_IN_PATH):
os.makedirs(DATA_IN_PATH )<save_to_csv>
model_roor_dir = '2nd-training-1st-gen-bert-download-from-gdrive'
MOD_DATA_STUCTURE = '2 split'
n_splits = 5
bert_trn_5fold, bert_val_5fold, bert_tst_5fold = get_nn_all_outputs('BERT' )
np.save(open(DATA_IN_PATH + TRAIN_INPUT_DATA, 'wb'), train_inputs)
 np.save(open(DATA_IN_PATH + TRAIN_LABEL_DATA, 'wb'), train_labels)
 clean_train_df.to_csv(DATA_IN_PATH + TRAIN_CLEAN_DATA, index = False)
 json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS, 'w'), ensure_ascii=False )<load_from_csv>
tokenizer = GPT2Tokenizer.from_pretrained('.. /input/gq-manual-uploads/gpt2 config from colab/')
 USING_PAD_TOKEN = False
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 save_file(outputs, 'GPT2_outputs')
 save_file(inputs, 'GPT2_inputs')
 save_file(test_inputs, 'GPT2_test_inputs' )
test_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv.zip", delimiter='\t')
 test_data.head()<string_transform>
model_roor_dir = 'gq-gpt2-pretrained'
MOD_DATA_STUCTURE = '2 split'
n_splits = 10
gpt2_trn, gpt2_val, gpt2_tst = get_nn_all_outputs('GPT2' )
clean_test_reviews = []
for review in test_data['review']:
clean_test_reviews.append(preprocess(review, remove_stopwords = True))
clean_test_df = pd.DataFrame({'review': clean_test_reviews, 'id': test_data['id']})
 test_id = np.array(test_data['id'])
 text_sequences = tokenizer.texts_to_sequences(clean_test_reviews)
 test_inputs = pad_sequences(text_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post' )<save_to_csv>
model_roor_dir = '2nd-training-1st-gen-gpt-download-from-gdrive'
MOD_DATA_STUCTURE = '2 split'
n_splits = 5
gpt2_trn_5fold, gpt2_val_5fold, gpt2_tst_5fold = get_nn_all_outputs('GPT2' )
TEST_INPUT_DATA = 'test_input.npy'
TEST_CLEAN_DATA = 'test_clean.csv'
TEST_ID_DATA = 'test_id.npy'
np.save(open(DATA_IN_PATH + TEST_INPUT_DATA, 'wb'), test_inputs)
 np.save(open(DATA_IN_PATH + TEST_ID_DATA, 'wb'), test_id)
 clean_test_df.to_csv(DATA_IN_PATH + TEST_CLEAN_DATA, index = False )<import_modules>
if USE_LGB:
def remove_articles(df):
for i in ['question_title', 'question_body', 'answer']:
df.loc[:,f'{i}_orig'] = df.loc[:,i]
for i in ['question_title', 'question_body', 'answer']:
df.loc[:,i] = df.loc[:,i].apply(lambda x: x.replace(' the ',' ' ).replace(' a ',' ' ).replace(' an ',' '))
return df
df_train = remove_articles(df_train)
 df_test = remove_articles(df_test )
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, GlobalAveragePooling1D, Dense, LSTM, Dropout, Bidirectional<define_variables>
if USE_LGB:
df_train.loc[:,'q_users_host'] = df_train.apply(lambda x: x.question_user_name + x.host, axis=1)
 df_train.loc[:,'a_users_host'] = df_train.apply(lambda x: x.answer_user_name + x.host, axis=1)
 df_test.loc[:,'q_users_host'] = df_test.apply(lambda x: x.question_user_name + x.host, axis=1)
 df_test.loc[:,'a_users_host'] = df_test.apply(lambda x: x.answer_user_name + x.host, axis=1)
 q_users_train = dict(df_train.q_users_host.value_counts())
 q_users_test = dict(df_test.q_users_host.value_counts())
 a_users_train = dict(df_train.a_users_host.value_counts())
 a_users_test = dict(df_test.a_users_host.value_counts())
 q_users = q_users_train
for i in q_users:
if i in q_users_test:
q_users_train[i] += q_users_test[i]
for i in q_users_test:
if i not in q_users:
q_users[i] = q_users_test[i]
a_users = a_users_train
for i in a_users:
if i in a_users_test:
a_users_train[i] += a_users_test[i]
for i in a_users_test:
if i not in a_users:
a_users[i] = a_users_test[i]
vocab_size = len(word_vocab)+1
vocab_size<choose_model_class>
if USE_LGB:
word_categories = ['adjectives','verbs','nouns','list_maker','digits','modals','posessives','persionals','interjection','direction','past_verb']
adjectives = ['JJ','JJR','JJS','RB','RBR','RBS']
verbs = ['VB','VBD','VBG','VBN','VBP','VBZ']
nouns = ['NN','NNS','NNP','NNPS']
list_maker = ['LS']
digits = ['CD']
modals = ['MD']
posessives = ['PPR$', 'POS']
persionals = ['PRP']
interjection = ['UH']
direction = ['TO']
past_verb = ['VBD','VBN']
def get_string_stats(df, var):
df.loc[:, f'{var}_numchars'] = df[var].apply(lambda x: len(x))
df.loc[:, f'{var}_numwords'] = df[var].apply(lambda x: len(x.split()))
df.loc[:, f'{var}_exclam_count'] = df[var].apply(lambda x: x.count('!'))
df.loc[:, f'{var}_question_count'] = df[var].apply(lambda x: x.count('?'))
df.loc[:, f'{var}_coma_count'] = df[var].apply(lambda x: x.count(','))
df.loc[:, f'{var}_dot_count'] = df[var].apply(lambda x: x.count('.'))
df.loc[:, f'{var}_all_punct_count'] = df[f'{var}_question_count']+df[f'{var}_coma_count']+df[f'{var}_exclam_count']
df.loc[:, f'{var}_all_punct_to_sentences'] = df.loc[:, f'{var}_all_punct_count']/df.loc[:, f'{var}_dot_count']
df.loc[:, f'{var}_questions_to_sentences'] = df.loc[:, f'{var}_question_count']/df.loc[:, f'{var}_dot_count']
df.loc[:, f'{var}_questions_to_words'] = df.loc[:, f'{var}_question_count']/df.loc[:, f'{var}_numwords']
df.loc[:, f'{var}_average_word_len'] = df[f'{var}_numchars']/df[f'{var}_numwords']
df.loc[:, f'{var}_capital_count'] = df[var].apply(lambda x: sum(1 for c in x if c.isupper()))
df.loc[:, f'{var}_capital_prop'] = df[f'{var}_capital_count']/df[f'{var}_numwords']
df.loc[:, f'{var}_other_ref'] = df[var].apply(lambda x: sum([x.count(i)for i in [' it ',' they ', " it's ", ' their ']]))
df.loc[:, f'{var}_self_ref'] = df[var].apply(lambda x: sum([x.count(i)for i in [' I ',' me ', " mine ", ' my ']]))
df.loc[:, f'{var}_total_ref'] = df[f'{var}_self_ref'] + df[f'{var}_other_ref']
df.loc[:, f'{var}_total_ref_prop'] = df.loc[:, f'{var}_total_ref']/df.loc[:, f'{var}_numwords']
df.loc[:, f'{var}_self_ref_prop'] = df[f'{var}_self_ref']/df[f'{var}_total_ref']
df.loc[:, f'{var}_other_ref_prop'] = df[f'{var}_other_ref']/df[f'{var}_total_ref']
df.loc[:, f'{var}_words_per_sentence'] = df[f'{var}_numwords']/df[f'{var}_dot_count']
df.loc[:, f'{var}_unique_words'] = df[f'{var}'].apply(lambda x: len(set(str(x ).split())))
 df.loc[:, f'{var}_unique_words_prop'] = df.loc[:, f'{var}_unique_words']/df.loc[:, f'{var}_numwords']
new_cols = [f'{var}_total_ref_prop', f'{var}_questions_to_words',f'{var}_questions_to_sentences',f'{var}_all_punct_to_sentences', f'{var}_unique_words_prop', f'{var}_unique_words', f'{var}_numchars',f'{var}_numwords',f'{var}_exclam_count',f'{var}_question_count',f'{var}_coma_count',f'{var}_dot_count',f'{var}_all_punct_count',f'{var}_average_word_len',f'{var}_capital_count',f'{var}_capital_prop',f'{var}_other_ref',f'{var}_self_ref',f'{var}_total_ref',f'{var}_self_ref_prop',f'{var}_other_ref_prop',f'{var}_words_per_sentence']
for category in word_categories:
df.loc[:, f'{var}_{category}'] = 0
new_cols.append(f'{var}_{category}')
 for idx in tqdm(range(df.shape[0]), total = df.shape[0]):
tokens = nltk.word_tokenize(df.loc[idx, var])
 tags = nltk.pos_tag(tokens)
 tags = [i[1] for i in tags]
for category in word_categories:
count = 0
for tag_name in globals() [category]:
count += tags.count(tag_name)
 df.loc[idx, f'{var}_{category}'] = count/df.loc[idx, f'{var}_numwords'] if df.loc[idx, f'{var}_numwords'] != 0 else 0
return df, new_cols
def get_extra_features_and_map(df):
df, nc1 = get_string_stats(df, 'question_title')
 df, nc2 = get_string_stats(df, 'question_body')
 df, nc3 = get_string_stats(df, 'answer')
 df.loc[:,'q_user_q_count'] = df.q_users_host.apply(lambda x: q_users[x] if x in q_users else 0)
 df.loc[:,'q_user_a_count'] = df.q_users_host.apply(lambda x: a_users[x] if x in a_users else 0)
 df.loc[:,'a_user_a_count'] = df.a_users_host.apply(lambda x: a_users[x] if x in a_users else 0)
 df.loc[:,'a_user_q_count'] = df.a_users_host.apply(lambda x: q_users[x] if x in q_users else 0)
 df.loc[:,'q_user_both_count'] = df.loc[:,'q_user_q_count'] + df.loc[:,'q_user_a_count']
df.loc[:,'a_user_both_count'] = df.loc[:,'a_user_a_count'] + df.loc[:,'a_user_q_count']
other_features = []
df.loc[:,'q_to_a_all_punct_count'] =(df.loc[:,'question_body_all_punct_count']+df.loc[:,'question_title_all_punct_count'])/df.loc[:,'answer_all_punct_count']
df.loc[:,'q_to_a_numwords'] =(df.loc[:,'question_body_numwords']+df.loc[:,'question_title_numwords'])/df.loc[:,'answer_numwords']
df.loc[:,'q_to_a_capital_count'] =(df.loc[:,'question_body_capital_count']+df.loc[:,'question_title_capital_count'])/df.loc[:,'answer_capital_count']
df.loc[:,'q_to_a_unique_words_prop'] =(df.loc[:,'question_body_unique_words_prop']+df.loc[:,'question_title_unique_words_prop'])/df.loc[:,'answer_unique_words_prop']
df.loc[:,'q_to_a_total_ref'] =(df.loc[:,'question_body_total_ref']+df.loc[:,'question_title_total_ref'])/df.loc[:,'answer_total_ref']
df.loc[:,'common_words'] = df.apply(lambda x: len(set(x.question_body.split() ).intersection(set(x.answer.split()))), axis=1)
 other_features += ['q_to_a_all_punct_count', 'q_to_a_numwords', 'q_to_a_capital_count', 'common_words', 'q_to_a_unique_words_prop', 'q_to_a_total_ref']
for category in word_categories:
df.loc[:,f'q_to_a_{category}'] = df.loc[:,f'question_body_{category}']/df.loc[:,f'answer_{category}']
other_features.append(f'q_to_a_{category}')
 df.loc[:,'spell_words'] = df.loc[:,'question_title'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['spell','spelled','spelt','spelling','write','wrote','written']))
df.loc[:,'spell_words'] += df.loc[:,'question_body'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['spell','spelled','spelt','spelling','write','wrote','written']))
df.loc[:,'compare_words'] = df.loc[:,'question_title'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['better','best','worse','nicer']))
df.loc[:,'compare_words'] += df.loc[:,'question_body'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['better','best','worse','nicer']))
df.loc[:,'consequence_words'] = df.loc[:,'question_title'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['if','when','will','would']))
df.loc[:,'consequence_words'] += df.loc[:,'question_body'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['if','when','will','would']))
other_features.append('spell_words')
 other_features.append('compare_words')
 other_features.append('consequence_words')
 onehots=[]
for i in df.loc[:,'category'].unique() :
df.loc[:,f'{i}_onehot'] = 0
df.loc[df.loc[:,'category']==i,f'{i}_onehot'] = 1
onehots.append(f'{i}_onehot')
 for i in df.loc[:,'host'].unique() :
df.loc[:,f'{i}_H_onehot'] = 0
df.loc[df.loc[:,'host']==i,f'{i}_H_onehot'] = 1
onehots.append(f'{i}_H_onehot')
 other_features = other_features+nc1+nc2+nc3+onehots+['q_user_q_count', 'q_user_a_count','a_user_a_count','a_user_q_count','q_user_both_count','a_user_both_count']
return df, other_features
df_train, other_features_train = get_extra_features_and_map(df_train)
 df_test, other_features = get_extra_features_and_map(df_test)
 for i in [a for a in other_features if a not in other_features_train]:
df_train.loc[:,i] = np.zeros(df_train.shape[0] )
def small_model() :
model = Sequential()
model.add(Embedding(vocab_size, 16))
model.add(GlobalAveragePooling1D())
 model.add(Dense(16, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 return model<find_best_params>
if USE_LGB:
def get_uids_all(df):
df.loc[:,'answer_uid'] = df.loc[:,'answer_user_page'].apply(lambda x: int(x.split('/')[-1]))
df.loc[:,'question_uid'] = df.loc[:,'question_user_page'].apply(lambda x: int(x.split('/')[-1]))
for idx in range(df.shape[0]):
split = [i for i in df.loc[idx,'url'].split('/')if i.isdigit() ]
df.loc[idx,'url_uid'] = int(split[-1])if len(split)>0 else -1
return df
df_train = get_uids_all(df_train)
 df_test= get_uids_all(df_test )
model = small_model()<train_model>
if USE_LGB:
se_path = ".. /input/stackexchange-data"
se_posts = pd.read_parquet(se_path+"/stackexchange_posts.parquet.gzip", engine='fastparquet')
 def get_post_info_se(df):
new_other_features = []
new_features = ['Score','ViewCount','AnswerCount','CommentCount','FavoriteCount','Tags']
df = df.merge(se_posts.loc[:,['Id', 'host', 'AcceptedAnswerId'] + new_features], how='left', left_on=['url_uid', 'host'], right_on=['Id', 'host'], sort=False)
 df.rename({i:'SE_QP_'+i for i in new_features}, inplace=True, axis=1)
 new_other_features += ['SE_QP_'+i for i in new_features]
return df, new_other_features
df_train, new_other_features = get_post_info_se(df_train)
 df_test, _ = get_post_info_se(df_test)
 del se_posts
gc.collect()
other_features += new_other_features
other_features.remove('SE_QP_Tags')
 all_tags = []
for i in range(df_train.shape[0]):
if(df_train.SE_QP_Tags.iloc[i] == None)or(pd.isna(df_train.SE_QP_Tags.iloc[i])) : continue
tags = df_train.SE_QP_Tags.iloc[i].replace('<','' ).split('>')
 all_tags += [t for t in tags if len(t)>0]
top_tags = list(pd.DataFrame(all_tags ).iloc[:,0].value_counts() [0:50].index)
 other_features += [f'tag_{i}' for i in top_tags]
for t in top_tags:
df_train.loc[:,f'tag_{t}'] = 0
df_test.loc[:,f'tag_{t}'] = 0
def parse_tags(df):
for i in range(df.shape[0]):
if(df.SE_QP_Tags.iloc[i] == None)or(pd.isna(df.SE_QP_Tags.iloc[i])) : continue
tags = df.SE_QP_Tags.iloc[i].replace('<','' ).split('>')
 tags = [t for t in tags if(len(t)> 0)&(t in top_tags)]
for t in tags:
df.loc[i,f'tag_{t}'] = 1
return df
df_train = parse_tags(df_train)
 df_test = parse_tags(df_test)
 se_path = ".. /input/stackexchange-data"
se_users = pd.read_parquet(se_path+"/stackexchange_users.parquet.gzip", engine='fastparquet')
 def get_user_info_se(df):
new_other_features = []
new_features = ['Reputation','Views','Upvotes','Downvotes']
df = df.merge(se_users.loc[:,['Id', 'host'] + new_features], how='left', left_on=['question_uid', 'host'], right_on=['Id', 'host'], sort=False)
 df.rename({i:'SE_Q_'+i for i in new_features}, inplace=True, axis=1)
 new_other_features += ['SE_Q_'+i for i in new_features]
new_features = ['Reputation','Views','Upvotes','Downvotes']
df = df.merge(se_users.loc[:,['Id', 'host'] + new_features], how='left', left_on=['answer_uid', 'host'], right_on=['Id', 'host'], sort=False)
 df.rename({i:'SE_A_'+i for i in new_features}, inplace=True, axis=1)
 new_other_features += ['SE_A_'+i for i in new_features]
return df, new_other_features
df_train, new_other_features = get_user_info_se(df_train)
 df_test, _ = get_user_info_se(df_test)
 del se_users
gc.collect()
other_features += new_other_features
history = model.fit(train_inputs,
train_labels,
epochs=10,
batch_size=256,
validation_split = 0.3 )<drop_column>
if USE_LGB:
def is_answer_accepted(df):
for i in df.loc[(df.host=='stackoverflow.com')& ~(df.QPAGE_accepted_answer_id.isna()),:].index.values:
df.loc[i,'answer_accepted'] = 1 if df.loc[i, 'answer_uid'] == df.loc[i, 'QPAGE_accepted_answer_id'] else 0
for i in df.loc[(df.host!='stackoverflow.com')& ~(df.AcceptedAnswerId.isna()),:].index.values:
df.loc[i,'answer_accepted'] = 1 if df.loc[i, 'answer_uid'] == df.loc[i, 'AcceptedAnswerId'] else 0
return df
df_train = is_answer_accepted(df_train)
 df_test = is_answer_accepted(df_test)
 other_features.append('answer_accepted' )
del model
del history<choose_model_class>
if USE_LGB:
class Base_Model(object):
def __init__(self, train_df, test_df, features, categoricals=[], n_splits=5, verbose=True, target=None, predict_test=True):
self.train_df = train_df
self.test_df = test_df
self.features = features
self.n_splits = 10
self.categoricals = categoricals
self.target = target
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.predict_test = predict_test
self.tst_pred, self.score, self.model, self.val_ys, self.val_preds = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_cv(self):
cv = GroupKFold(n_splits=self.n_splits)
 return cv.split(X=self.train_df.question_body_orig, groups=self.train_df.question_body_orig)
 def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def fit(self):
oof_pred = []
oof_ys_all = []
y_pred = np.zeros(( len(self.test_df),))
for fold,(train_idx, val_idx)in enumerate(self.cv):
if fold < ACTUAL_FOLDS:
x_train, x_val = self.train_df[self.features].iloc[train_idx], self.train_df[self.features].iloc[val_idx]
y_train, y_val = self.train_df[self.target][train_idx], self.train_df[self.target][val_idx]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model = self.load_model(fold)
 conv_x_val = self.convert_x(x_val.reset_index(drop=True))
preds_all = model.predict(conv_x_val)
 preds_all = 1/(1 + np.exp(-preds_all))
oof_pred += preds_all.tolist()
if self.predict_test:
x_test = self.convert_x(self.test_df[self.features])
 current_test_preds = model.predict(x_test ).reshape(y_pred.shape)
 current_test_preds = 1/(1 + np.exp(-current_test_preds))
current_test_preds = current_test_preds/ACTUAL_FOLDS
y_pred += current_test_preds
if self.verbose:print('Partial score(all)of fold {} is: {}'.format(fold, spearmanr(y_val, preds_all ).correlation))
oof_ys_all += list(y_val.reset_index(drop=True ).values)
 loss_score = spearmanr(oof_ys_all, oof_pred ).correlation
if self.verbose:
print('Our oof cohen kappa score(all)is: ', loss_score)
 return y_pred, loss_score, model, np.array(oof_ys_all), np.array(oof_pred)
 class Lgb_Model(Base_Model):
def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
return lgb.train(self.params, train_set, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 def load_model(self, fold):
model = pickle.load(open(f'.. /input/gq-lgb/{self.target}/{self.target}_{fold}.p', 'rb'))
return model
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {'n_estimators':5000,
'boosting_type': 'gbdt',
'objective': 'cross_entropy_lambda',
'subsample': 0.75,
'subsample_freq': 1,
'learning_rate': 0.01,
'feature_fraction': 0.8,
'max_depth': 150,
'num_leaves': 50,
'lambda_l1': 0.1,
'lambda_l2': 0.1,
'early_stopping_rounds': 300,
'min_data_in_leaf': 1,
'min_gain_to_split': 0.01,
'max_bin': 400
}
return params
class Xgb_Model(Base_Model):
def load_model(self, fold):
model = pickle.load(open(f'.. /input/gq-xgb/{self.target}/{self.target}_{fold}.p', 'rb'))
return model
def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
return xgb.train(self.params, train_set,
num_boost_round=5000, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=verbosity, early_stopping_rounds=100)
 def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = xgb.DMatrix(x_train, y_train)
 val_set = xgb.DMatrix(x_val, y_val)
 return train_set, val_set
def convert_x(self, x):
return xgb.DMatrix(x)
 def get_params(self):
params = {
'objective':'reg:logistic',
'n_estimators':5000,
'max_depth':12,
'eta':0.05}
return params
def big_model() :
model = Sequential()
model.add(Embedding(vocab_size, 16))
model.add(Bidirectional(LSTM(64, recurrent_dropout=0.1)))
 model.add(Dropout(0.5))
model.add(Dense(32, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(1, activation = 'sigmoid'))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 return model<choose_model_class>
if USE_LGB:
one_lgb_model = pickle.load(open(f'.. /input/gq-lgb/question_opinion_seeking/question_opinion_seeking_0.p', 'rb'))
lgb_pretrained_features = one_lgb_model.feature_name()
for i in lgb_pretrained_features:
if i not in other_features:
print(f'{i} not in other features here, adding zeros')
 df_train.loc[:, i] = np.zeros(df_train.shape[0])
 df_test.loc[:, i] = np.zeros(df_test.shape[0] )
model = big_model()<train_model>
%%time
if USE_LGB:
ACTUAL_FOLDS = 3
lgb_val_scores = []
n_output_categories = len(output_categories)
 lgb_val_outputs_all = []
lgb_val_preds_all = []
lgb_tst_preds_all = []
for idx, i in enumerate(output_categories, 1):
lgb_model = Lgb_Model(df_train, df_test, lgb_pretrained_features, target=i, verbose=False)
 lgb_val_outputs_all.append(lgb_model.val_ys)
 lgb_val_preds_all.append(lgb_model.val_preds)
 lgb_tst_preds_all.append(lgb_model.tst_pred)
 lgb_val_scores.append(lgb_model.score)
 print(f'{idx}/{n_output_categories}',i, lgb_model.score)
 save_file(lgb_val_outputs_all, 'lgb_val_outputs_all')
 save_file(lgb_val_preds_all, 'lgb_val_preds_all')
 save_file(lgb_tst_preds_all, 'lgb_tst_preds_all')
 save_file(df_train, 'df_train')
 save_file(df_test, 'df_test')
 else:
lgb_val_outputs_all = [0]*30
lgb_val_preds_all = [0]*30
lgb_tst_preds_all = [0]*30
history = model.fit(train_inputs,
train_labels,
epochs=10,
batch_size=256,
validation_split = 0.3 )<save_model>
if USE_XGB:
one_xgb_model = pickle.load(open(f'.. /input/gq-xgb/question_opinion_seeking/question_opinion_seeking_0.p', 'rb'))
xgb_pretrained_features = one_xgb_model.feature_names
for i in xgb_pretrained_features:
if i not in other_features:
print(f'{i} not in other features here, adding zeros')
 df_train.loc[:, i] = np.zeros(df_train.shape[0])
 df_test.loc[:, i] = np.zeros(df_test.shape[0] )
model.save('my_model.h5' )<predict_on_test>
%%time
if USE_XGB:
ACTUAL_FOLDS = 3
xgb_val_scores = []
n_output_categories = len(output_categories)
 xgb_val_outputs_all = []
xgb_val_preds_all = []
xgb_tst_preds_all = []
for idx, i in enumerate(output_categories, 1):
xgb_model = Xgb_Model(df_train, df_test, xgb_pretrained_features, target=i, verbose=False)
 xgb_val_outputs_all.append(xgb_model.val_ys)
 xgb_val_preds_all.append(xgb_model.val_preds)
 xgb_tst_preds_all.append(xgb_model.tst_pred)
 xgb_val_scores.append(xgb_model.score)
 print(f'{idx}/{n_output_categories}',i, xgb_model.score)
 save_file(xgb_val_outputs_all, 'xgb_val_outputs_all')
 save_file(xgb_val_preds_all, 'xgb_val_preds_all')
 save_file(xgb_tst_preds_all, 'xgb_tst_preds_all')
 save_file(df_train, 'df_train')
 save_file(df_test, 'df_test')
 else:
xgb_val_outputs_all = [0]*30
xgb_val_preds_all = [0]*30
xgb_tst_preds_all = [0]*30
%%time
test_label = model.predict_classes(test_inputs )<save_to_csv>
def get_rounding(ys, preds):
rounding_types = ['Normal', 'Ceil', 'Floor']
rounding_funcs = [np.round, np.ceil, np.floor]
dec_places = [1,2,3,4,5]
score = spearmanr(ys, preds ).correlation
if np.isnan(score): score=-100
best_result = {'Type':'No rounding','DP':0, 'func':None}
for r_type, r_func in zip(rounding_types, rounding_funcs):
for dp in dec_places:
if r_type == 'Normal':
rounded_preds = r_func(preds, dp)
 cur_score = spearmanr(ys, rounded_preds ).correlation
else:
rounded_preds = r_func(preds*(10**dp)) /(10**dp)
 cur_score = spearmanr(ys, rounded_preds ).correlation
if np.isnan(cur_score): cur_score = 0
if cur_score > score:
score = cur_score
best_result['Type'] = r_type
best_result['DP'] = dp
best_result['func'] = r_func
return score, best_result
def submit(predictions):
test_data['sentiment'] = predictions
test_data.to_csv('submission.csv', index=False, columns=['id','sentiment'])
 submit(test_label )<import_modules>
def inverse_spearman_r(weights, ys, preds):
mixed_val_preds = np.array([i*w for i,w in zip(preds, weights)] ).sum(axis=0)
 score = spearmanr(ys, mixed_val_preds ).correlation
if np.isnan(score): score=-100
return -score
def optimize_mixing_weights(ys, preds):
naive_mix = np.array(preds ).mean(axis=0)
 score = spearmanr(ys, naive_mix ).correlation
if np.isnan(score): score=-100
c_dict = {'type':'eq','fun':lambda x: 1-np.sum(x)}
optim_func = partial(inverse_spearman_r, ys=ys, preds=preds)
 x0 = np.array([1/len(preds)]*len(preds))
res = minimize(optim_func, x0, method='SLSQP', constraints=c_dict)
 print(f'Best score {res.fun}; weights {res.x}')
 return res.x
def optimize_mixing_weights_guessing(ys, preds):
variants = [0.0, 0.01, 0.04, 0.1, 0.12, 0.16, 0.18, 0.2, 0.22, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.82, 0.85, 0.88, 0.9, 0.94, 0.95]
models_n = len(preds)
 weights = [1/models_n]*models_n
best_weights = weights
best_score = 0
for comb in combinations_with_replacement(variants, models_n):
weights = list(comb)
 if np.sum(weights)!= 1: continue
score = spearmanr(ys, np.array([i*w for i,w in zip(preds, weights)] ).sum(axis=0)).correlation
if np.isnan(score): score=-100
if score>best_score: best_weights, best_score = weights.copy() , score
print(f'Best score {best_score}; weights {best_weights}')
 return best_weights
import numpy as np
import pandas
import tensorflow as tf
import re
import keras
from keras.preprocessing.sequence import pad_sequences
from keras.engine.topology import Layer
from keras.layers import *
from keras import backend as K
from keras.models import Model
import gensim
from gensim.models import Word2Vec
import math<define_variables>
val10_common = [np.where(val10==i)[0][0] for i in common_validation_idx]
val5_common = [np.where(val5==i)[0][0] for i in common_validation_idx]
ct_map = {"ain't": "is not", "'cause": "because","he's": "he is", "how'd": "how did","how's": "how is",
"'m": " am", "let's": "let us", "ma'am": "madam", "o'clock": "of the clock","shan't": "shall not",
"so's": "so as", "this's": "this is", "that's": "that is", "there's": "there is",
"here's": "here is", "what's": "what is", "when's": 'when is', "where'd": "where did",
"where's": "where is", "who's": "who is", "why's": "why is", "y'all": "you all", "'d": ' would',
"'ll": ' will', "n't": " not", "'ve": " have", "'re": ' are'}<string_transform>
val10_fold0_common = [np.where(val10_fold0==i)[0] for i in common_validation_idx]
val10_fold0_common = [i[0] for i in val10_fold0_common if len(i)>0]
def CleanSentence(string):
for a, b in ct_map.items() :
string = string.replace(a, b)
 string = string.replace('!', ' puncexc')
 string = string.replace('?', ' puncask')
 string = re.sub(r'[0-9]+', '0', string)
 string = string.replace(' ', '9')
 string = ''.join(list(filter(str.isalnum, string)))
 return string.split('9' )<string_transform>
%%time
weights_all = []
weights_for_rounding = []
scores_all = []
df_sub = pd.read_csv(PATH+'sample_submission.csv')
 bestonly_raw_scores = []
best_rounded_scores = []
best_weighted_scores = []
if USE_SAMPLE: df_sub=df_sub.iloc[0:round(0.1*df_sub.shape[0]),:]
if USE_LGB:
if len(lgb_val_scores)!= 30:
USE_LGB=False
print('Something wrong with LGB, switching it off')
 for idx, cat in enumerate(output_categories):
lgb_score = spearmanr(xlnet_val[1][:,idx], lgb_val_preds_all[idx] ).correlation if USE_LGB else np.nan
xgb_score = spearmanr(xlnet_val[1][:,idx], xgb_val_preds_all[idx] ).correlation if USE_XGB else np.nan
gpt2_score = spearmanr(gpt2_val[1][:,idx], gpt2_val[0][:,idx] ).correlation
bert_score = spearmanr(bert_val[1][:,idx], bert_val[0][:,idx] ).correlation
xlnet_score = spearmanr(xlnet_val[1][:,idx], xlnet_val[0][:,idx] ).correlation
bert_score_3split = spearmanr(bert_val_3split[1][:,idx], bert_val_3split[0][:,idx] ).correlation if USE_3SPLIT_BERT else np.nan
bert_score_5fold = spearmanr(bert_val_5fold[1][:,idx], bert_val_5fold[0][:,idx] ).correlation
gpt2_score_5fold = spearmanr(gpt2_val_5fold[1][:,idx], gpt2_val_5fold[0][:,idx] ).correlation
scores_all.append([lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold])
 selected_models_scores = [i for i in [lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold] if not np.isnan(i)]
selected_models_val_preds = [i for i, s in zip([lgb_val_preds_all[idx], xgb_val_preds_all[idx], gpt2_val[0][:,idx], bert_val[0][:,idx], xlnet_val[0][:,idx], bert_val_3split[0][:,idx], bert_val_5fold[0][:,idx], gpt2_val_5fold[0][:,idx]],[lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold])if not np.isnan(s)]
selected_models_tst_preds = [i for i, s in zip([lgb_tst_preds_all[idx], xgb_tst_preds_all[idx], gpt2_tst[:,idx], bert_tst[:,idx], xlnet_tst[:,idx], bert_tst_3split[:,idx], bert_tst_5fold[:,idx], gpt2_tst_5fold[:,idx]],[lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold])if not np.isnan(s)]
current_best = np.max(selected_models_scores)
 bestonly_raw_scores.append(current_best)
 if USE_LGB & USE_XGB:
val_preds_10and5 = [lgb_val_preds_all[idx][val10_common], xgb_val_preds_all[idx][val10_common], gpt2_val[0][val10_common,idx], bert_val[0][val10_common,idx], xlnet_val[0][val10_common,idx], bert_val_5fold[0][val5_common,idx], gpt2_val_5fold[0][val5_common,idx]]
scores10and5 = np.sum([np.exp(i*beta)for i in [lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]])
 weights10and5 = [np.exp(i*beta)/scores10and5 for i in [lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]]
if USE_LGB &(not USE_XGB):
val_preds_10and5 = [lgb_val_preds_all[idx][val10_common], gpt2_val[0][val10_common,idx], bert_val[0][val10_common,idx], xlnet_val[0][val10_common,idx], bert_val_5fold[0][val5_common,idx], gpt2_val_5fold[0][val5_common,idx]]
scores10and5 = np.sum([np.exp(i*beta)for i in [lgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]])
 weights10and5 = [np.exp(i*beta)/scores10and5 for i in [lgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]]
if(not USE_LGB)&(not USE_XGB):
val_preds_10and5 = [gpt2_val[0][val10_common,idx], bert_val[0][val10_common,idx], xlnet_val[0][val10_common,idx], bert_val_5fold[0][val5_common,idx], gpt2_val_5fold[0][val5_common,idx]]
scores10and5 = np.sum([np.exp(i*beta)for i in [gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]])
 weights10and5 = [np.exp(i*beta)/scores10and5 for i in [gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]]
val_mix = np.array([i*w for i,w in zip(val_preds_10and5, weights10and5)] ).sum(axis=0)
 common_val = xlnet_val[1][val10_common,idx]
weights_for_rounding.append(weights10and5)
 if not ROUND: print(f'{idx} {cat} scores: LGB: {lgb_score}, GPT2: {gpt2_score}, BERT: {bert_score}, XLNET: {xlnet_score}')
 if WEIGHTS == 'softmax':
all_scores = np.sum(np.exp([i*beta for i in selected_models_scores]))
weights = [np.exp(i*beta)/all_scores for i in selected_models_scores]
best_weighted_scores.append(spearmanr(common_val, np.array([i*w for i,w in zip(val_preds_10and5, weights)] ).sum(axis=0)).correlation)
 weights_all.append(weights)
 df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, weights)] ).sum(axis=0)
 if WEIGHTS == 'dynamic':
weights = optimize_mixing_weights_guessing(common_val, val_preds_10and5)
 weights_all.append(weights)
 best_weighted_scores.append(spearmanr(common_val, np.array([i*w for i,w in zip(val_preds_10and5, weights)] ).sum(axis=0)).correlation)
 val_mix = np.array([i*w for i,w in zip(val_preds_10and5, weights)] ).sum(axis=0)
 df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, weights)] ).sum(axis=0)
 if WEIGHTS == 'dynamic softmax':
ds_score = -100
best_beta = None
for b in np.arange(1,30,0.5):
all_scores = np.sum(np.exp([i*b for i in selected_models_scores]))
try_weights = [np.exp(i*b)/all_scores for i in selected_models_scores]
try_mix = np.array([i*w for i,w in zip(val_preds_10and5, try_weights)] ).sum(axis=0)
 try_score = spearmanr(common_val, try_mix ).correlation
if try_score > ds_score:
best_beta = b
ds_score = try_score
weights = try_weights.copy()
val_mix = try_mix.copy()
weights_all.append(weights)
 best_weighted_scores.append(ds_score)
 print(f'Best beta: {best_beta}')
 df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, weights)] ).sum(axis=0)
 if WEIGHTS == 'bestonly':
df_sub.iloc[:, idx+1] = selected_models_tst_preds[np.argmax(selected_models_scores)]
if ROUND:
best_rounded_score, rounding_method = get_rounding(common_val, val_mix)
 best_rounded_scores.append(best_rounded_score)
 print(f"{idx} {cat}: mixed score {best_rounded_score}; {rounding_method['Type']}, {rounding_method['DP']}")
 unrounded_backup = df_sub.iloc[:, idx+1].copy()
if rounding_method['Type'] == 'Normal':
df_sub.iloc[:, idx+1] = rounding_method['func'](df_sub.iloc[:, idx+1], rounding_method['DP'])
 elif(rounding_method['Type'] == 'Ceil')or(rounding_method['Type'] == 'Floor'):
df_sub.iloc[:, idx+1] = rounding_method['func'](df_sub.iloc[:, idx+1]*(10**rounding_method['DP'])) /(10**rounding_method['DP'])
 if USE_TEST_SAFETY_ADJUSTMENTS:
if df_sub.iloc[:,idx+1].var() == 0:
print('Test predictions are STILL homogenous, reverting to softmax weights')
 all_scores = np.sum(np.exp([i for i in selected_models_scores]))
softmax_weights = [np.exp(i)/all_scores for i in selected_models_scores]
df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, softmax_weights)] ).sum(axis=0)
 if df_sub.iloc[:,idx+1].var() == 0:
raise Exception('var = 0')
 if df_sub.iloc[:,idx+1].min() < 0:
raise Exception("<0")
 if df_sub.iloc[:,idx+1].max() > 1:
raise Exception(">1")
 if df_sub.isna().sum().sum() != 0:
raise Exception("na in sub")
 df_sub.to_csv('submission.csv', index=False )
def TextToSequence(data, vocab):
return [list(filter(lambda x: x is not None, [vocab[token].index + 1 if token in vocab.keys() else None
for token in sentence])) for sentence in data]<load_from_csv>
save_file(weights_all, 'weights_all')
 save_file(weights_for_rounding, 'weights_all')
 save_file(scores_all, 'weights_all' )
def textpre(emb_dim):
train = pandas.read_csv('.. /input/word2vec-nlp-tutorial/labeledTrainData.tsv','\t')
 unlabel_train = pandas.read_csv('.. /input/word2vec-nlp-tutorial/unlabeledTrainData.tsv','\t',
error_bad_lines = False)
 additional_data = pandas.read_csv('.. /input/imdb-dataset-of-50k-movie-reviews/IMDB Dataset.csv')
 test = pandas.read_csv('.. /input/word2vec-nlp-tutorial/testData.tsv','\t')
 s = train['sentiment']
del train['sentiment']
train['sentiment'] = s
unlabel_train['sentiment'] = -2
additional_data['sentiment'] = np.asarray(additional_data['sentiment'] == 'positive', dtype = 'int32')
 s = np.asarray(np.asarray(test['id'].apply(lambda x: x.split('_')[1]), dtype = 'int32')> 5,
dtype = 'int32')
 test['sentiment'] = -1
del train['id'], test['id'], unlabel_train['id']
data = pandas.concat([train, additional_data, unlabel_train, test])
 review = list(data['review'].apply(lambda x: CleanSentence(x)))
 model = Word2Vec(review, size = emb_dim, workers = 4, iter = 0)
 emb_mat = np.concatenate([np.zeros(( 1, emb_dim), dtype = 'float32'), model.wv.vectors])
 print(emb_mat.shape)
 seq = TextToSequence(review, model.wv.vocab)
 seq = pad_sequences([doc if len(doc)<= 128 else doc[:64] + doc[-64:] for doc in seq], padding = 'post')
 tar = np.asarray(data['sentiment'], dtype = 'int32')
 r = data['sentiment'] >= 0
train = np.hstack(( seq[r], np.expand_dims(tar[r], -1)))
 r = data['sentiment'] == -1
test = np.hstack(( seq[r], np.expand_dims(s, -1)))
 print('text preprocessing completed')
 return [train, test, emb_mat]<choose_model_class>
np.max(df_sub.iloc[:,1:].to_numpy().flatten()), np.min(df_sub.iloc[:,1:].to_numpy().flatten()), df_sub.shape
class Self_Attention(Layer):
def __init__(self, head_num, head_size, **kwargs):
self.head_size = head_size
self.sqrt_head_size = math.sqrt(head_size)
 self.head_num = head_num
self.output_units = head_size * head_num
self.supports_masking = True
super(Self_Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
self.d_model = input_shape[-1]
self.kernal = self.add_weight('kernal',(self.head_num, 3, self.d_model, self.head_size),
initializer = 'glorot_uniform')
 super(Self_Attention, self ).build(input_shape)
 self.built = True
def call(self, inputs, mask = None):
w = K.dot(inputs, self.kernal)
 w = K.permute_dimensions(w, [3, 2, 0, 1, 4])
 Head = K.batch_dot(w[0], K.permute_dimensions(w[1], [0, 1, 3, 2]))
if mask is not None:
new_mask = K.expand_dims(mask, -1)
 new_mask = K.batch_dot(new_mask, K.permute_dimensions(new_mask, [0, 2, 1]))
new_mask = K.expand_dims(new_mask, 0)
 Head -= 1e8 *(K.ones_like(Head)- new_mask)
 Head = K.softmax(Head)
 Head = K.batch_dot(Head, w[2])
 Head = K.permute_dimensions(Head, [1, 2, 3, 0])
 return K.reshape(Head,(-1, K.shape(Head)[1],self.output_units))
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[1], self.output_units)
 def compute_mask(self, inputs, mask = None):
pass<choose_model_class>
np.mean(bestonly_raw_scores), np.min(bestonly_raw_scores), np.max(bestonly_raw_scores), len(bestonly_raw_scores )
class Position_Embedding(Layer):
def __init__(self, **kwargs):
self.supports_masking = True
super(Position_Embedding, self ).__init__(**kwargs)
 def build(self, input_shape):
self.depth = input_shape[1]
self.d_model = input_shape[2]
super(Position_Embedding, self ).build(input_shape)
 self.built = True
def call(self, inputs):
pos_emb = np.zeros([1, self.depth, self.d_model])
 p = np.zeros([self.d_model], dtype = 'int32')
 for i in range(self.d_model):
p[i] = math.pow(10000,(i - 1)/ self.d_model)if i%2 else math.pow(10000, i / self.d_model)
 for i in range(self.depth):
for j in range(self.d_model):
pos_emb[0][i][j] = math.cos(i / p[j])if j%2 else math.sin(i / p[j])
 return inputs + tf.Variable(np.asarray(pos_emb, dtype = 'float32'), trainable = False)
 def compute_mask(self, inputs, mask = None):
pass<categorify>
np.mean(best_weighted_scores), np.min(best_weighted_scores), np.max(best_weighted_scores), len(best_weighted_scores )
def Attention_block(inputs, head_num, head_size, hidden_units):
s = Self_Attention(head_num, head_size )(inputs)
 s = BatchNormalization()(Add()([s, inputs]))
output = Dense(hidden_units, activation = 'relu' )(s)
 output = Dense(int(inputs.shape[-1]))(output)
 return BatchNormalization()(Add()([s, output]))<train_on_grid>
np.mean(best_rounded_scores), np.min(best_rounded_scores), np.max(best_rounded_scores), len(best_rounded_scores )
train, test, emb_mat = textpre(256)
 tf.reset_default_graph()
keras.backend.clear_session()
np.random.seed(7)
 tf.set_random_seed(7)
 x = Input(shape =(train.shape[1] - 1,), dtype = 'int32')
 emb = Embedding(emb_mat.shape[0], emb_mat.shape[1], weights = [emb_mat], mask_zero = True,
trainable = False )(x)
 prob = Position_Embedding()(BatchNormalization()(emb))
prob = Attention_block(prob, 4, 64, 1024)
 prob = GlobalMaxPool1D()(prob)
 prob = Dense(1, activation = 'sigmoid' )(prob)
 model = Model(inputs = x, outputs = prob)
 model.compile('adam', 'binary_crossentropy', ['accuracy'])
 model.summary()
model.fit(train[:, :-1], train[:, -1], 128, 12, validation_data =(test[:, :-1], test[:, -1]))
res = np.asarray(model.predict(test[:, :-1])> 0.5, dtype = 'int32')
 p = pandas.read_csv('.. /input/word2vec-nlp-tutorial/sampleSubmission.csv')
 p['sentiment'] = res[:, 0]
p.to_csv('submission.csv', index = False )<set_options>
[(c,np.round(s,3)) for c,s in zip(output_categories, scores_all)]
<set_options><EOS> 
[(c,np.round(s,3)) for c,s in zip(output_categories, weights_all)]