Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<load_from_csv>
BS = 8
SEED = 42
training_dataset = pd.read_csv('.. /input/donorschoose-application-screening/train.csv', sep=',')
 resources_dataset = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv', sep=',')
 test_dataset = pd.read_csv('.. /input/donorschoose-application-screening/test.csv', sep=',' )<load_from_csv>
!pip install.. /input/sacremoses/sacremoses-master/ > /dev/null
!pip install.. /input/transformers/transformers-master/ > /dev/null
dictionary_dataset = pd.read_csv('.. /input/english-word-frequency/unigram_freq.csv', sep=',')
 with open('.. /input/unix-words/words/en')as f:
content = f.readlines()
` at the end of each line
content = [x.strip() for x in content]
unix_set = set(content)
 USING_BROWN = False
if USING_BROWN:
if '/usr/share/nltk_data' in nltk.data.path:
nltk.data.path.remove('/usr/share/nltk_data')
 nltk.data.path.append('.. /input/brown-corpus/brown')
 nltk.data.path
brown = LazyCorpusLoader('brown', CategorizedTaggedCorpusReader, r'c[a-z]\d\d',
cat_file='cats.txt', tagset='brown', encoding="ascii",
nltk_data_subdir='brown-corpus/brown')
 list_of_lists_of_tuples = brown.tagged_sents()
list_sents = []
for list_of_tuples in list_of_lists_of_tuples:
sentence = " ".join([tupl[0] for tupl in list_of_tuples])
 list_sents.append(sentence)
 list_sents
pd_brown = pd.DataFrame({'sentence': list_sents})
 pd_brown['sentence'] = pd_brown['sentence'].replace('(\-)|(/)|(\.\.\.\.) |(\.\.\.) ', ' ', regex=True)
 pd_brown['sentence'] = pd_brown['sentence'].replace('["
pd_brown['sentence'] = pd_brown['sentence'].replace('() |() ', ' ', regex=True)
 pd_brown['sentence'] = pd_brown['sentence'].str.lower()
pd_brown[0:10]<count_missing_values>
tqdm.pandas()
device = torch.device('cuda' )
resources_dataset.isnull().any()<filter>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv' ).fillna(' ')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv' ).fillna(' ')
 sub = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv' ).fillna(' ' )
resources_dataset.loc[resources_dataset["description"].isnull() ]<count_missing_values>
def _get_segments_xlnet(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "<sep>":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return [0] *(max_seq_length - len(tokens)) + segments
def _get_segments_bert(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_xlnet(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = [5] *(max_seq_length-len(token_ids)) + token_ids
return input_ids
def _get_ids_bert(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length=512-1,
t_max_len=70-1, q_max_len=219, a_max_len=219):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = norm_token_length(q, q_new_len)
 a = norm_token_length(a, a_new_len)
 return t, q, a
def norm_token_length(tokens, l):
if len(tokens)> l:
head = l//2
tail = l - head
return tokens[:head] + tokens[-tail:]
else:
return tokens[:l]
def _convert_to_bert_inputs(title, question, answer, cate, pretrained_weights, max_sequence_length=512):

 if "bert-base" in pretrained_weights:
stoken = ["[CLS]"] + [cate] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_bert(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_bert(stoken, max_sequence_length)
 elif pretrained_weights == "xlnet-base-cased":
stoken = [cate] + title + ["<sep>"] + question + ["<sep>"] + answer + ["<sep>", "<cls>"]
input_ids = _get_ids_xlnet(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_xlnet(stoken, max_sequence_length)
 try:
cls_index = input_segments.index(5)- 1
except ValueError:
cls_index = -1
input_segments[cls_index] = 2
return [input_ids, input_segments]
def convert_row(row, pretrained_weights):
if pretrained_weights == "bert-base-uncased":
c = f"[{row['category'].lower() }]"
elif pretrained_weights == "bert-base-cased":
c = f"[{row['category']}]"
elif pretrained_weights == "xlnet-base-cased":
c = f"[{row['category']}]"
t, q, a = title = row["question_title"], row["question_body"], row["answer"]
t, q, a = _trim_input(t, q, a)
 ids, segments = _convert_to_bert_inputs(t, q, a, c, pretrained_weights)
 return np.array([[ids, segments]] )
resources_dataset = resources_dataset.fillna({'description' : 'no_detail'})
 resources_dataset.isnull().any()<feature_engineering>
class CustomBert(BertPreTrainedModel):
def __init__(self, config):
super(CustomBert, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.classifier = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 pooled_output = torch.cat([outputs[2][-1*i][:,0] for i in range(1, self.n_use_layer+1)], dim=1)
 pooled_output = self.dense1(pooled_output)
 pooled_output = self.dense2(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
resources_dataset['cost'] = resources_dataset['quantity'] * resources_dataset['price']
resources_dataset[0:9]<groupby>
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bert-swa/')
 tokenizer.added_tokens_encoder
grouped_ids = resources_dataset.groupby(['id'], as_index=False)
 resources_condensed = grouped_ids.agg({'cost' : 'sum'} ).rename(columns={'cost' : 'total_cost'})
 resources_condensed[69060:69065]<groupby>
pretrained_weights = 'bert-base-uncased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )
group_the_ids = resources_dataset.groupby(['id'], as_index=False)
 all_condensed = grouped_ids.agg({'description' : lambda x: ' '.join(x),
'quantity' : 'sum',
'cost' : 'sum'} ).rename(columns={'description' : 'full_description',
'quantity' : 'total_quantity',
'cost' : 'total_cost'})
 all_condensed[69060:69065]<filter>
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()
entry = all_condensed.loc[all_condensed['id'] == 'p069063'].reset_index()
entry.loc[0, 'full_description']<merge>
model_dir_target = ".. /input/quest-bertuncased-10fold/"
bert_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-uncased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 bert_pred_lst.append(test_pred )
combined_training_dataset = pd.merge(training_dataset, all_condensed, on='id')
 combined_training_dataset[0:5]<merge>
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bertcased-tokenizer/')
 tokenizer.added_tokens_encoder
combined_test_dataset = pd.merge(test_dataset, all_condensed, on='id')
 combined_test_dataset[0:5]<categorify>
pretrained_weights = 'bert-base-cased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )
combined_training_dataset['teacher_prefix'] = combined_training_dataset['teacher_prefix'].fillna('none')
 combined_test_dataset['teacher_prefix'] = combined_test_dataset['teacher_prefix'].fillna('none')
 combined_training_dataset = combined_training_dataset.fillna('')
 combined_test_dataset = combined_test_dataset.fillna('' )<feature_engineering>
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()
combined_training_dataset.loc[combined_training_dataset['project_essay_4'] != '', 'project_essay_1'] = combined_training_dataset['project_essay_1'] + ' ' + combined_training_dataset['project_essay_2']
combined_training_dataset.loc[combined_training_dataset['project_essay_4'] != '', 'project_essay_2'] = combined_training_dataset['project_essay_3'] + ' ' + combined_training_dataset['project_essay_4']
combined_test_dataset.loc[combined_test_dataset['project_essay_4'] != '', 'project_essay_1'] = combined_test_dataset['project_essay_1'] + ' ' + combined_test_dataset['project_essay_2']
combined_test_dataset.loc[combined_test_dataset['project_essay_4'] != '', 'project_essay_2'] = combined_test_dataset['project_essay_3'] + ' ' + combined_test_dataset['project_essay_4']
combined_training_dataset[16:20]<filter>
model_dir_target = ".. /input/quest-bertcased-10fold/"
cased_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 cased_pred_lst.append(test_pred )
combined_training_dataset.loc[18, 'project_essay_2']<feature_engineering>
class CustomXLNet(XLNetForSequenceClassification):
def __init__(self, config):
super(CustomXLNet, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.n_labels = config.num_labels
self.transformer = XLNetModel(config)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.logits_proj = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(
 self,
input_ids=None,
attention_mask=None,
mems=None,
perm_mask=None,
target_mapping=None,
token_type_ids=None,
input_mask=None,
head_mask=None,
inputs_embeds=None,
labels=None, 
):
transformer_outputs = self.transformer(
 input_ids,
attention_mask=attention_mask,
mems=mems,
perm_mask=perm_mask,
target_mapping=target_mapping,
token_type_ids=token_type_ids,
input_mask=input_mask,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 output = torch.cat([transformer_outputs[1][-1*(i+1)][:,-1] for i in range(self.n_use_layer)], dim=1)
 output = self.dense1(output)
 output = self.dense2(output)
 logits = self.logits_proj(output)
 outputs =(logits,)+ transformer_outputs[1:]
return outputs
USING_DICTIONARY_CODE = True
if USING_DICTIONARY_CODE == False:
the_essays = [
'project_essay_1',
'project_essay_2',
'full_description',
'project_resource_summary'
]
def tidy_essays(dataset):
for col_name in the_essays:
dataset[col_name] = dataset[col_name].replace(r'(\\r)|(\ 
)', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(r' ', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(r' ', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(r' ', ' ', regex=True)
 tidy_essays(combined_training_dataset)
 tidy_essays(combined_test_dataset)
 text_columns = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary',
'project_subject_categories',
'project_subject_subcategories',
'full_description'
]
punc_pattern = '["
def text_edits(dataset):
for col_name in text_columns:
dataset[col_name] = dataset[col_name].str.lower()
dataset[col_name] = dataset[col_name].replace(r'!', ' !', regex=True)
 dataset[col_name] = dataset[col_name].replace(r'(\-)|(/)', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(punc_pattern, '', regex=True)
 text_edits(combined_training_dataset)
 text_edits(combined_test_dataset)
 combined_training_dataset[16:22]<string_transform>
tokenizer = XLNetTokenizer.from_pretrained('.. /input/quest-xlnet-tokenizer/')
 tokenizer.added_tokens_encoder
if USING_DICTIONARY_CODE:
text_columns = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary',
'project_subject_categories',
'project_subject_subcategories',
'full_description'
]
def tidy_text(dataset):
for col_name in text_columns:
dataset[col_name] = dataset[col_name].replace('\?', '', regex=True)
 dataset[col_name] = dataset[col_name].str.encode('ascii', 'replace')
 dataset[col_name] = dataset[col_name].str.decode('utf8')
 dataset[col_name] = dataset[col_name].replace('([A-Z][.])+', '', regex=True)
 dataset[col_name] = dataset[col_name].str.lower()
in the text:
dataset[col_name] = dataset[col_name].replace('(\\\\r)|(\\\ 
)', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.com[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.org[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.gov[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.edu[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.net[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('
dataset[col_name] = dataset[col_name].replace('\\\', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('(\-)|(/)|(\.\.\.\.) |(\.\.\.) ', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace('=', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('(?<=\w)[,.;!"](?=\w)', '=', regex=True)
 dataset[col_name] = dataset[col_name].replace('!', ' ! ', regex=True)
 dataset[col_name] = dataset[col_name].replace('["
dataset[col_name] = dataset[col_name].replace(' * *', ' ', regex=True)
 dataset[col_name] = dataset[col_name].str.strip()
return
tidy_text(combined_training_dataset)
 tidy_text(combined_test_dataset )<feature_engineering>
pretrained_weights = "xlnet-base-cased"
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )
if USING_DICTIONARY_CODE:
COLUMNS_TO_CHECK = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary'
]
def total_typing_errors(df):
new_col_names = []
for col in COLUMNS_TO_CHECK:
new_col_name = col + '_typing_errors'
new_col_name = new_col_name[8:]
df[new_col_name] = df[col].str.count('=')
 new_col_names.append(new_col_name)
 df['total_typing_errors'] = df[new_col_names].sum(axis=1)
 return
total_typing_errors(combined_training_dataset)
 total_typing_errors(combined_test_dataset)
 for col_name in text_columns:
combined_training_dataset[col_name] = combined_training_dataset[col_name].replace('=', ' ', regex=True)
 combined_test_dataset[col_name] = combined_test_dataset[col_name].replace('=', ' ', regex=True )<string_transform>
model = CustomXLNet.from_pretrained(".. /input/quest-xlnet-tokenizer/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()
if USING_DICTIONARY_CODE:
join_cols = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary',
'full_description'
]
all_dataset = pd.concat([combined_training_dataset[join_cols], combined_test_dataset[join_cols]])
 def get_unique_word_sets(df):
new_set = set()
df.str.split().apply(new_set.update)
 return new_set
title_set = get_unique_word_sets(all_dataset['project_title'])
 essay_1_set = get_unique_word_sets(all_dataset['project_essay_1'])
 essay_2_set = get_unique_word_sets(all_dataset['project_essay_2'])
 summary_set = get_unique_word_sets(all_dataset['project_resource_summary'])
 description_set = get_unique_word_sets(all_dataset['full_description'])
 print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))
print(len(description_set))<define_variables>
model_dir_target = ".. /input/quest-xlnetcased-10fold/"
xlnet_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/xlnet-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids != 5 ).int().to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 xlnet_pred_lst.append(test_pred )
if USING_DICTIONARY_CODE:
dictionary_set = set(dictionary_dataset['word'])
 dictionary_set.add('!')
 print(len(dictionary_set))<define_variables>
def sigmoid(x):
return 1.0 /(1.0 + np.exp(-x))
bert_pred = np.array(bert_pred_lst ).mean(0)
 cased_pred = np.array(cased_pred_lst ).mean(0)
 xlnet_pred = np.array(xlnet_pred_lst ).mean(0)
 bert_pred = sigmoid(bert_pred)
 cased_pred = sigmoid(cased_pred)
 xlnet_pred = sigmoid(xlnet_pred)
 test_pred = bert_pred*0.4 + cased_pred*0.2 + xlnet_pred*0.4
if USING_DICTIONARY_CODE:
title_set = title_set - dictionary_set - unix_set
essay_1_set = essay_1_set - dictionary_set - unix_set
essay_2_set = essay_2_set - dictionary_set - unix_set
summary_set = summary_set - dictionary_set - unix_set
description_set = description_set - dictionary_set - unix_set
print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))
print(len(description_set))<define_variables>
lgbm_models = pickle.load(open(".. /input/quest-lgbm/lgbm_question_type_spelling.pkl", 'rb'))
count_vectorizers = pickle.load(open(".. /input/quest-lgbm/tfidf_vectorizers.pkl", 'rb'))
if USING_DICTIONARY_CODE:
title_set = title_set - description_set
essay_1_set = essay_1_set - description_set
essay_2_set = essay_2_set - description_set
summary_set = summary_set - description_set
print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))<normalization>
dfs = []
for idx, col_name in enumerate(["question_title", "question_body", "answer"]):
X = count_vectorizers[idx].transform(test[col_name])
 feat = [f"{col_name}_{c}".encode("utf-8")for c in count_vectorizers[idx].get_feature_names() ]
df = pd.DataFrame(X.toarray() , columns=feat)
 dfs.append(df)
 test_x = pd.concat(dfs, axis=1)
 test_x.shape
if USING_DICTIONARY_CODE:
def make_set_alpha(word_set):
words_to_drop = set()
for word in word_set:
if word.isalpha() == False:
words_to_drop.add(word)
 return word_set - words_to_drop
title_set = make_set_alpha(title_set)
 essay_1_set = make_set_alpha(essay_1_set)
 essay_2_set = make_set_alpha(essay_2_set)
 summary_set = make_set_alpha(summary_set)
 print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))<string_transform>
lgbm_pred = np.zeros(len(test_x))
for fold in range(4):
lgbm_pred += lgbm_models[fold].predict(test_x)/4
lgbm_pred.shape
if USING_DICTIONARY_CODE:
COLS_AND_SETS = {
'project_title' : title_set,
'project_essay_1' : essay_1_set,
'project_essay_2' : essay_2_set,
'project_resource_summary' : summary_set
}
def count_spelling_errors(string, word_set):
list_words = string.split()
error_count = 0
for word in list_words:
if word in word_set:
error_count += 1
return error_count
def total_spelling_errors(df):
new_col_names = []
for col, word_set in COLS_AND_SETS.items() :
new_col_name = col + '_spelling_errors'
new_col_name = new_col_name[8:]
df[new_col_name] = df[col].apply(count_spelling_errors, args=(word_set,))
new_col_names.append(new_col_name)
 df['total_spelling_errors'] = df[new_col_names].sum(axis=1)
 return
total_spelling_errors(combined_training_dataset)
 total_spelling_errors(combined_test_dataset )<filter>
def spearman_corr(y_true, y_pred):
if np.ndim(y_pred)== 2:
corr = np.nan_to_num([stats.spearmanr(y_true[:, i], y_pred[:, i])[0] for i in range(y_true.shape[1])] ).mean()
else:
corr = stats.spearmanr(y_true, y_pred)[0]
return corr
class OptimizedRounder(object):
def __init__(self,
n_overall: int = 5,
n_classwise: int = 5,
n_classes: int = 7,
metric: str = "qwk"):
self.n_overall = n_overall
self.n_classwise = n_classwise
self.n_classes = n_classes
self.coef = [1.0 / n_classes * i for i in range(1, n_classes)]
self.metric_str = metric
self.metric = spearman_corr
def _loss(self, X: np.ndarray, y: np.ndarray)-> float:
X_p = np.digitize(X, self.coef)
 ll = -self.metric(y, X_p)
 return ll
def fit(self, X: np.ndarray, y: np.ndarray):
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [ 
(0.01, 1.0 / self.n_classes + 0.05),
]
for i in range(1, self.n_classes):
ab_start.append(( i * 1.0 / self.n_classes + 0.05, 
(i + 1)* 1.0 / self.n_classes + 0.05))
for _ in range(self.n_overall):
for idx in range(self.n_classes - 1):
a, b = ab_start[idx]
self.coef[idx] = a
la = self._loss(X, y)
 self.coef[idx] = b
lb = self._loss(X, y)
 for it in range(self.n_classwise):
if la > lb:
a = b -(b - a)* golden1
self.coef[idx] = a
la = self._loss(X, y)
 else:
b = b -(b - a)* golden2
self.coef[idx] = b
lb = self._loss(X, y)
 def predict(self, X: np.ndarray)-> np.ndarray:
X_p = np.digitize(X, self.coef)
 return X_p
if USING_DICTIONARY_CODE:
combined_training_dataset.loc[combined_training_dataset['total_spelling_errors'] > 12]<filter>
optR_lst = pickle.load(open(".. /input/quest-optimizedrounder/optR_lst_10fold_ensemble_v3.pkl", 'rb'))
lst = []
for idx, optR in enumerate(optR_lst):
coeff = optR.predict(test_pred[:, idx])
 lst.append(coeff/norm_dict[sub.columns[1:][idx]])
 opt_preds = np.array(lst ).T
test_pred = opt_preds
test_pred[:,19] = lgbm_pred
combined_training_dataset['project_essay_2'].loc[110575]<filter>
test_pred[test[test["category"] != "CULTURE"].index, 19] = 0.0
test["host_info"] = test["question_user_page"].map(lambda x: x.split("/")[2].replace(".stackexchange.com", ""))
test_pred[test[test["host_info"].map(lambda x: x not in ["english", "ell"])].index, 19] = 0.0
<filter><EOS> 
sub[sub.columns[1:]] = test_pred
sub.to_csv("submission.csv", index=False)
 sub.head()