Upload 2 files

requirements.txt

@@ -1,4 +1,5 @@
 altair==5.1.1
+annoy==1.17.3
 attrs==23.1.0
 blinker==1.6.2
 cachetools==5.3.1

utils.py

@@ -11,6 +11,7 @@ import time
 from gensim.corpora import Dictionary
 from gensim.models import KeyedVectors, TfidfModel
 from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix, WordEmbeddingSimilarityIndex
+from gensim.similarities.annoy import AnnoyIndexer
 from io import BytesIO
 from nltk import pos_tag, word_tokenize
 from nltk.corpus import stopwords, wordnet
@@ -274,9 +275,8 @@ def performStemming(text):
 
 @st.cache_data
 def loadModel():
-    # model_path = '~/Projects/hau/csstudy/final-csstudy/wiki-news-300d-1M-subword.vec'
     model_path = 'wiki-news-300d-1M-subword.vec'
-    model = KeyedVectors.load_word2vec_format(model_path, limit = 100000)
+    model = KeyedVectors.load_word2vec_format(model_path)
     return model
 
 model = loadModel()
@@ -295,7 +295,10 @@ def rankResumes(text, df):
     progressBar.progress(25, text = "Creating a TF-IDF model ...")
     tfidf = TfidfModel(dictionary = dictionary)
     progressBar.progress(38, text = "Creating a Similarity Index...")
-    similarityIndex = WordEmbeddingSimilarityIndex(model)
+    words = [word for word, count in dictionary.most_common()]
+    wordVectors = model.vectors_for_all(words, allow_inference = False)
+    indexer = AnnoyIndexer(wordVectors, num_trees = 10)
+    similarityIndex = WordEmbeddingSimilarityIndex(wordVectors, kwargs = {'indexer': indexer})
     progressBar.progress(50, text = "Creating a Similarity Matrix...")
     similarityMatrix = SparseTermSimilarityMatrix(similarityIndex, dictionary, tfidf)
     progressBar.progress(63, text = "Setting up job description as the query ...")
@@ -307,8 +310,8 @@ def rankResumes(text, df):
     )
     similarities = index[query]
     progressBar.progress(88, text = "Finishing touches ...")
-    df['Similarity Score'] = similarities
-    df['Rank'] = df['Similarity Score'].rank(ascending=False, method='dense').astype(int)
+    df['Similarity Score (-1 to 1)'] = similarities
+    df['Rank'] = df['Similarity Score (-1 to 1)'].rank(ascending=False, method='dense').astype(int)
     df.sort_values(by='Rank', inplace=True)
     df.drop(columns = ['cleanedResume'], inplace = True)
     endTime = time.time()
@@ -317,7 +320,7 @@ def rankResumes(text, df):
     minutes, _ = divmod(remainder, 60)
     secondsWithDecimals = '{:.2f}'.format(elapsedSeconds % 60)
     elapsedTimeStr = f'{hours} h : {minutes} m : {secondsWithDecimals} s'
-    progressBar.progress(100, text = f'Classification Complete!')
+    progressBar.progress(100, text = f'Ranking Complete!')
     time.sleep(1)
     progressBar.empty()
     st.info(f'Finished ranking {len(df)} resumes - {elapsedTimeStr}')