Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
num_raters_dict = {'question_asker_intent_understanding': 18,
'question_body_critical': 18,
'question_conversational': 6,
'question_expect_short_answer': 6,
'question_fact_seeking': 6,
'question_has_commonly_accepted_answer': 6,
'question_interestingness_others': 18,
'question_interestingness_self': 18,
'question_multi_intent': 6,
'question_not_really_a_question': 6,
'question_opinion_seeking': 6,
'question_type_choice': 6,
'question_type_compare': 6,
'question_type_consequence': 6,
'question_type_definition': 6,
'question_type_entity': 6,
'question_type_instructions': 6,
'question_type_procedure': 6,
'question_type_reason_explanation': 6,
'question_type_spelling': 3,
'question_well_written': 18,
'answer_helpful': 18,
'answer_level_of_information': 18,
'answer_plausible': 18,
'answer_relevance': 18,
'answer_satisfaction': 30,
'answer_type_instructions': 6,
'answer_type_procedure': 6,
'answer_type_reason_explanation': 6,
'answer_well_written': 18
}<feature_engineering>
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from datetime import datetime
from sklearn.neighbors import KNeighborsClassifier
def submssion_trick(output):
predictions = np.zeros_like(output)
 for i in range(output.shape[0]):
for j in range(output.shape[1]):
col = output_categories[j]
num_raters = num_raters_dict[col]
if j == 19:
if i not in culture_category_list:
predictions[i][j] = 0.0
else:
predictions[i][j] = np.floor(output[i][j] * num_raters)/ num_raters
else:
predictions[i][j] = np.floor(output[i][j] * num_raters)/ num_raters
if num_raters == 18:
predictions[i][j] = max(predictions[i][j], 1/3)
 if num_raters == 30:
predictions[i][j] = max(predictions[i][j], 1/5)
 return predictions<normalization>
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv")
 sample = pd.read_csv('.. /input/sample_submission.csv')
scaler = MinMaxScaler()
for i in range(30):
output[:, i] = scaler.fit_transform(output[:, i].reshape(-1, 1)).reshape(-1, )<concatenate>
Y = train_data['type']
train_data = train_data.drop(['id'], axis=1)
 train_data = train_data.drop(['type'], axis=1)
 test_data = test_data.drop(['id'], axis=1)
 print(train_data.columns)
 print(test_data.columns )
predictions = submssion_trick(output )<load_from_csv>
def exploreData(data, column):
return data[column].value_counts()
def imputation(data, column, value):
data.loc[data[column].isnull() , column] = value
def countMissing(data):
missing = data.columns[data.isnull().any() ].tolist()
return missing
submission = pd.read_csv(PATH +'sample_submission.csv')
 for i in range(30):
if len(np.unique(predictions[:, i])) == 1:
print(i)
 print(submission.columns[i+1])
 print('before', np.unique(predictions[:, i]))
print('after', np.unique(predictions[:, i]))<save_to_csv>
cat_cols = ['color']
for col in cat_cols:
train_data = pd.concat(( train_data, pd.get_dummies(train_data[col], prefix=col)) , axis=1)
 del train_data[col]
test_data = pd.concat(( test_data, pd.get_dummies(test_data[col], prefix=col)) , axis=1)
 del test_data[col]
submission.loc[:, 'question_asker_intent_understanding':] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head(10 )<set_options>
train_data =(train_data - train_data.min())/(train_data.max() - train_data.min())
 test_data =(test_data - test_data.min())/(test_data.max() - test_data.min() )
warnings.filterwarnings("ignore" )<load_pretrained>
print(Y.value_counts() )
vector_dim = 300
encoder = hub.load("/kaggle/input/universalsentenceencoderqa" )<define_search_model>
Y = [0 if y == 'Ghoul' else 1 if y == 'Goblin' else 2 for y in Y]
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer('english' )<define_variables>
rfe = RandomForestRegressor(n_estimators = 500)
 rfe.fit(train_data, Y)
 imp_score = rfe.feature_importances_
imp_score = pd.DataFrame({'feature': train_data.columns, 'score': imp_score})
 print(imp_score.sort_values(by = 'score', ascending = False))
contractions = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'd've": "he would have",
"he'll": "he will",
"he'll've": "he he will have",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"i'd've": "I would have",
"i'll": "I will",
"i'll've": "I will have",
"i'm": "I am",
"i've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have"
}
rules = {
"'t": " not",
"'cause": " because",
"'ve": " have",
"'t": " not",
"'s": " is",
"'d": " had"
}
punctuations = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']',
'>', '%', '=', '
'®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█',
'½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶',
'↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',
'⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪',
'╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√'] + list(string.punctuation)
 spell_replace = {
'usepackage':'latex',
'orf19':'gene',
'documentclass':'latex',
'magento':'open-source e-commerce',
'appium':'web-app',
'tikz':'programming language',
'tikzpicture':'programming language',
'openvpn':'vpn',
'httpclient':'http client',
'arraylist':'array list',
'jsonobject': 'json',
'artifactid':'xml',
'hwnd':'os'
}
special_chars = ",." : )(- ! ? | ; ' $ & / [ ] > % =<load_from_csv>
neighbors = [1, 2, 5, 10]
param = []
scores = []
start = datetime.now()
print('start time: ', start)
 for n in neighbors:
knn = KNeighborsClassifier(n_neighbors = n)
 scores.append(cross_val_score(knn, train_data, Y, scoring="accuracy", cv = 10 ).mean())
 param.append(n)
 finish = datetime.now()
secs =(finish-start ).seconds
print('elapsed time in minutes: ', secs/60.0)
 scores = pd.DataFrame({'parameter': param, 'score': scores})
 print(scores.sort_values(by = 'score', ascending = False))
train_df = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 test_df = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 cols = train_df.columns[11:]
train_df['combined_t_q'] = train_df[['question_title', 'question_body']].apply(lambda x: x['question_title']+' '+x['question_body'], axis = 1)
 test_df['combined_t_q'] = test_df[['question_title', 'question_body']].apply(lambda x: x['question_title']+' '+x['question_body'], axis = 1)
 train_df['combined_t_a'] = train_df[['question_title', 'answer']].apply(lambda x: x['question_title']+' '+x['answer'], axis = 1)
 test_df['combined_t_a'] = test_df[['question_title', 'answer']].apply(lambda x: x['question_title']+' '+x['answer'], axis = 1 )<count_unique_values>
knn = KNeighborsClassifier(n_neighbors = 5)
 knn.fit(train_data, Y)
 res = knn.predict(test_data )
final_outputs = train_df[cols].values
unique_outputs = [np.sort(train_df[col].unique())[np.newaxis, :] for col in cols]<categorify>
type = ['Ghoul' if r == 0 else 'Goblin' if r == 1 else 'Ghost' for r in res]
type = pd.Series(type)
 print(type.value_counts() )
def preprocess_text(s):
s = s.lower()
for key, value in contractions.items() :
s = s.replace(key, f' {value} ')
 for key, value in rules.items() :
s = s.replace(key, f' {value} ')
 for punct in punctuations:
if punct in special_chars:
s = s.replace(punct, f' {punct} ')
 else:
s = s.replace(punct, ' ')
 for key, value in spell_replace.items() :
s = s.replace(key, value)
 s = re.sub('\s+', ' ', s)
 return s<feature_engineering>
sample = pd.read_csv('.. /input/sample_submission.csv')
 id = sample['id']
df = pd.DataFrame({'type': type})
 df['id'] = id
df = df[['id', 'type']]
df.columns
print(df.head(5))
df['type'].value_counts()
train_df['clean_answer'] = train_df['answer'].apply(preprocess_text)
 train_df['clean_t_q'] = train_df['combined_t_q'].apply(preprocess_text)
 test_df['clean_answer'] = test_df['answer'].apply(preprocess_text)
 test_df['clean_t_q'] = test_df['combined_t_q'].apply(preprocess_text )<count_unique_values>
df.to_csv('knn_submission.csv', index=False)
 print(os.listdir('.'))
all_texts = train_df.clean_answer.tolist() + train_df.clean_t_q.tolist() + test_df.clean_answer.tolist() + test_df.clean_t_q.tolist()
all_texts = [text.split() for text in all_texts]
counter = Counter()
for text in all_texts:
counter.update(text )<feature_engineering>
alphas = [0.01, 0.03, 0.1, 0.3, 1.0, 10]
regs = ["l1","l2"]
scores_1 = []
param = []
scores_2 = []
start = datetime.now()
print('start time: ', start)
 for alpha in alphas:
for reg in regs:
lm1 = OneVsOneClassifier(linear_model.LogisticRegression(C = alpha, multi_class = 'multinomial', solver = 'saga', penalty = reg))
lm2 = OneVsRestClassifier(linear_model.LogisticRegression(C = alpha, multi_class = 'multinomial', solver = 'saga', penalty = reg))
scores_1.append(cross_val_score(lm1, train_data, Y, scoring="accuracy", cv = 10 ).mean())
 scores_2.append(cross_val_score(lm2, train_data, Y, scoring="accuracy", cv = 10 ).mean())
 param.append([alpha, reg])
 finish = datetime.now()
secs =(finish-start ).seconds
print('elapsed time in minutes: ', secs/60.0)
 scores_1 = pd.DataFrame({'parameter': param, 'score': scores_1})
 scores_2 = pd.DataFrame({'parameter': param, 'score': scores_2})
 print('OneVsONe')
 print(scores_1.sort_values(by = 'score', ascending = False))
print('OneVsRest')
 print(scores_2.sort_values(by = 'score', ascending = False))
vocab = {}
vocab['token2id'] = {key:id+1 for id,(key, _)in enumerate(counter.items())}
vocab['id2token'] = {value:key for key, value in vocab['token2id'].items() }
vocab['word_freq'] = dict(counter )<categorify>
lm1 = OneVsOneClassifier(linear_model.LogisticRegression(C = 10, multi_class = 'multinomial', solver = 'saga', penalty = 'l1'))
lm1.fit(train_data, Y)
 res = lm1.predict(test_data)
 print(res )
def build_embedding_matrix(vocab, texts, embedd_size, model):
n = len(vocab['token2id'])+1
embedding_matrix = np.zeros(( n, embedd_size))
for text in texts:
for key in text:
word = key
try:
embedding_matrix[vocab['token2id'][word]] = model.wv[word]
continue
except:
pass
return embedding_matrix<train_model>
type = ['Ghoul' if r == 0 else 'Goblin' if r == 1 else 'Ghost' for r in res]
type = pd.Series(type)
 print(type.value_counts() )
model = Word2Vec(size=300, window = 5, min_count = 1)
 model.build_vocab(all_texts)
 total_examples = model.corpus_count
model.intersect_word2vec_format('/kaggle/input/fasttext/fasttext-wiki-news-subwords-300', lockf=1.0)
 model.train(all_texts, total_examples=total_examples, epochs=5)
 embedding_matrix = build_embedding_matrix(vocab, all_texts, 300, model )<string_transform>
df = pd.DataFrame({'type': type})
 df['id'] = id
df = df[['id', 'type']]
df.columns
print(df.head(5))
df['type'].value_counts()
def get_token_ids(texts, max_length):
tokens = []
for text in texts:
tmp_tokens = []
if len(text.split())> max_length:
for each in(text.split() [:(max_length//2)] + text.split() [-(max_length//2):]):
tmp_tokens.append(vocab['token2id'][each])
 tokens.append(tmp_tokens)
 else:
for each in(text.split() [:max_length]):
tmp_tokens.append(vocab['token2id'][each])
 tokens.append(tmp_tokens)
 return tf.keras.preprocessing.sequence.pad_sequences(tokens, padding="post", maxlen=max_length )<feature_engineering>
df.to_csv('onevsone_submission.csv', index=False)
 print(os.listdir('.'))
MAX_LENGTH = 250
data = {}
data['train_question_title'] = get_token_ids(train_df['clean_t_q'], MAX_LENGTH)
 data['train_answer'] = get_token_ids(train_df['clean_answer'], MAX_LENGTH)
 data['test_question_title'] = get_token_ids(test_df['clean_t_q'], MAX_LENGTH)
 data['test_answer'] = get_token_ids(test_df['clean_answer'], MAX_LENGTH )<categorify>
lm2 = OneVsRestClassifier(linear_model.LogisticRegression(C = 1.0, multi_class = 'multinomial', solver = 'saga', penalty = 'l1'))
lm2.fit(train_data, Y)
 res = lm2.predict(test_data)
 print(res )
data['train_question_title_use'] = []
data['train_answer_use'] = []
data['test_question_title_use'] = []
data['test_answer_use'] = []
BATCH_SIZE = 4
for i in range(0, train_df.shape[0], BATCH_SIZE):
data['train_question_title_use'] += [encoder.signatures['question_encoder'](tf.constant(train_df['clean_t_q'].iloc[i:i+BATCH_SIZE].tolist())) ['outputs'].numpy().astype(np.float16)]
data['train_answer_use'] += [encoder.signatures['response_encoder'](input = tf.constant(train_df['clean_answer'].iloc[i:i+BATCH_SIZE].tolist()),
context = tf.constant(train_df['clean_answer'].iloc[i:i+BATCH_SIZE].tolist())) ['outputs'].numpy().astype(np.float16)]
for i in range(0, test_df.shape[0], BATCH_SIZE):
data['test_question_title_use'] += [encoder.signatures['question_encoder'](tf.constant(test_df['clean_t_q'].iloc[i:i+BATCH_SIZE].tolist())) ['outputs'].numpy().astype(np.float16)]
data['test_answer_use'] += [encoder.signatures['response_encoder'](input = tf.constant(test_df['clean_answer'].iloc[i:i+BATCH_SIZE].tolist()),
context = tf.constant(test_df['clean_answer'].iloc[i:i+BATCH_SIZE].tolist())) ['outputs'].numpy().astype(np.float16)]
data['train_question_title_use'] = np.vstack(data['train_question_title_use'])
 data['train_answer_use'] = np.vstack(data['train_answer_use'])
 data['test_question_title_use'] = np.vstack(data['test_question_title_use'])
 data['test_answer_use'] = np.vstack(data['test_answer_use'] )<define_variables>
type = ['Ghoul' if r == 0 else 'Goblin' if r == 1 else 'Ghost' for r in res]
type = pd.Series(type)
 print(type.value_counts() )
def get_generator(X, batch_size = 32, training = True):
Y = X[1]
N = X[0][0].shape[0]
if training == True:
indexes = np.random.permutation(N)
 else:
indexes = np.arange(N)
 def generator() :
for i in indexes:
yield {"input_1": X[0][0][i], "input_2": X[0][1][i],"input_3": X[0][2][i], "input_4": X[0][3][i]}, Y[i]
return tf.data.Dataset.from_generator(generator,
output_types =({"input_1": tf.int32, "input_2": tf.int32,"input_3": tf.float16,"input_4": tf.float16}, tf.float16)).repeat().batch(batch_size)
 def SpearmanCorrCoeff(A, B):
overall_score = 0
for index in range(A.shape[1]):
overall_score += spearmanr(A[:, index], B[:, index] ).correlation
return np.round(overall_score/A.shape[1], 3)
 class PredictCallback(tf.keras.callbacks.Callback):
def __init__(self, data, labels):
self.data = data
self.labels = labels
def on_epoch_end(self, epoch, logs = {}):
predictions = self.model.predict(self.data)
 print('
Validation Score - ' + str(SpearmanCorrCoeff(self.labels, predictions)))
 class EWA(tf.keras.callbacks.Callback):
def on_train_batch_end(self, batch, logs = None):
global prev_weights
if prev_weights is None:
prev_weights = [tf.identity(x)for x in self.model.trainable_variables]
else:
beta = 0.1
for index, _ in enumerate(self.model.trainable_variables):
self.model.trainable_variables[index].assign(self.model.trainable_variables[index]*beta +(1-beta)*prev_weights[index])
 prev_weights = [tf.identity(x)for x in self.model.trainable_variables]<choose_model_class>
df = pd.DataFrame({'type': type})
 df['id'] = id
df = df[['id', 'type']]
df.columns
print(df.head(5))
df['type'].value_counts()
<count_unique_values><EOS> 
df.to_csv('onevsrest_submission.csv', index=False)
 print(os.listdir('.'))