Kaggle competition: NTUST: Information Retrieval and Applications
<SOS> metric: map@{k} Kaggle data source: ntust:-information-retrieval-and-applications<define_variables>
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer
FEATURES_BASELINE = ["Sex", "Pclass"]<string_transform>
root_path = '.. /input/homework4_v2/'
max_df = 0.95
min_df = 5
smooth_idf = True
sublinear_tf = True
alpha = 1
beta = 0.75
gamma = 0.15
rel_count = 5
nrel_count = 1
iters = 5
def title(name):
surname_split = name.split(", ")
 title_split = surname_split[1].split(".")
 title = title_split[0]
return title<feature_engineering>
with open(root_path + 'doc_list.txt')as file:
doc_list = [line.rstrip() for line in file]
with open(root_path + 'query_list.txt')as file:
query_list = [line.rstrip() for line in file]
data["Title"] = data.Name.apply(title)
 data['Title'] = data['Title'].replace(['Dona', 'Mlle', 'Ms'], 'Miss')
 data['Title'] = data['Title'].replace(['Lady', 'the Countess', 'Mme'], 'Mrs')
 data['Title'] = data['Title'].replace(['Jonkheer', 'Don', 'Sir', 'Capt', 'Major', 'Col'], 'Mr')
 data['Title'].value_counts()<feature_engineering>
documents, queries = [], []
for doc_name in doc_list:
with open(root_path + 'Document/' + doc_name)as file:
doc = ' '.join([word for line in file.readlines() [3:] for word in line.split() [:-1]])
 documents.append(doc)
 for query_name in query_list:
with open(root_path + 'Query/' + query_name)as file:
query = ' '.join([word for line in file.readlines() for word in line.split() [:-1]])
 queries.append(query )
data["Neighbors"] = neighbors(data["Ticket"] )<filter>
vectorizer = TfidfVectorizer(max_df=max_df, min_df=min_df,
smooth_idf=smooth_idf, sublinear_tf=sublinear_tf)
 doc_tfidfs = vectorizer.fit_transform(documents ).toarray()
query_vecs = vectorizer.transform(queries ).toarray()
cos_sim = cosine_similarity(query_vecs, doc_tfidfs)
 rankings = np.flip(cos_sim.argsort() , axis=1 )
print("Passengers with Embarked NaNs")
 print(data[pd.isnull(data.Embarked)])
 embarked_nans_fare = data.loc[(data.Pclass == 1)&(data.Neighbors == 1), "Fare"].median()
print(f"Median fare for this type of passengers: {embarked_nans_fare}" )<feature_engineering>
for _ in range(iters):
rel_vecs = doc_tfidfs[rankings[:, :rel_count]].mean(axis=1)
 nrel_vecs = doc_tfidfs[rankings[:, -nrel_count:]].mean(axis=1)
 query_vecs = alpha * query_vecs + beta * rel_vecs - gamma * nrel_vecs
cos_sim = cosine_similarity(query_vecs, doc_tfidfs)
 rankings = np.flip(cos_sim.argsort(axis=1), axis=1 )
<feature_engineering><EOS> 
with open('submission.csv', mode='w')as file:
file.write('Query,RetrievedDocuments
')
 for query_name, ranking in zip(query_list, rankings):
ranked_docs = ' '.join([doc_list[idx] for idx in ranking])
 file.write('%s,%s
' %(query_name, ranked_docs))