Upload 17 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wiki-news-300d-1M-subword.vec filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,99 @@
+import pandas as pd
+import streamlit as st 
+
+from utils import *
+
+backgroundPattern = """
+<style>
+[data-testid="stAppViewContainer"] {
+    background-color: #0E1117;
+    opacity: 1;
+    background-image: radial-gradient(#282C34 0.75px, #0E1117 0.75px);
+    background-size: 15px 15px;
+}
+</style>
+"""
+
+# backgroundPattern = """
+# <style>
+# [data-testid="stAppViewContainer"] {
+#     background-color: #FFFFFF;
+#     opacity: 1;
+#     background-image: radial-gradient(#D1D1D1 0.75px, #FFFFFF 0.75px);
+#     background-size: 15px 15px;
+# }
+# </style>
+# """
+
+st.markdown(backgroundPattern, unsafe_allow_html=True)
+
+st.write("""
+# Resume Screening & Classification
+""")
+st.caption("""
+Using K-Nearest Neighbors (KNN) algorithm and Cosine Similarity
+######
+""")
+
+tab1, tab2, tab3 = st.tabs(['Getting Started', 'Classify', 'Rank'])
+
+with tab1:
+    writeGettingStarted()
+
+with tab2:
+    st.header('Input')
+    uploadedResumeClf = st.file_uploader('Upload Resumes', type = 'xlsx', key = 'upload-resume-clf')
+
+    if uploadedResumeClf is not None:
+        isButtonDisabledClf = False
+    else:
+        st.session_state.processClf = False 
+        isButtonDisabledClf = True
+
+    if 'processClf' not in st.session_state:
+        st.session_state.processClf = False
+
+    st.button('Start Processing', on_click = clickClassify, disabled = isButtonDisabledClf, key = 'process-clf')
+
+    if st.session_state.processClf:
+        st.divider()
+        st.header('Output')
+        resumeClf = pd.read_excel(uploadedResumeClf)
+        resumeClf = classifyResumes(resumeClf)
+        with st.expander('View Bar Chart'):
+            barChart = createBarChart(resumeClf)
+            st.altair_chart(barChart, use_container_width = True)
+        currentClf = filterDataframeClf(resumeClf)
+        st.dataframe(currentClf, use_container_width = True, hide_index = True)
+        xlsxClf = convertDfToXlsx(currentClf)
+        st.download_button(label='Save Current Output as XLSX', data = xlsxClf, file_name = 'Resumes_categorized.xlsx')
+
+with tab3:
+    st.header('Input')
+    uploadedJobDescriptionRnk = st.file_uploader('Upload Job Description', type = 'txt', key = 'upload-jd-rnk')
+    uploadedResumeRnk = st.file_uploader('Upload Resumes', type = 'xlsx', key = 'upload-resume-rnk')
+
+    if all([uploadedJobDescriptionRnk, uploadedResumeRnk]):
+        isButtonDisabledRnk = False
+    else:
+        st.session_state.processRank = False
+        isButtonDisabledRnk = True
+
+    if 'processRank' not in st.session_state:
+        st.session_state.processRank = False
+
+    st.button('Start Processing', on_click = clickRank, disabled = isButtonDisabledRnk, key = 'process-rnk')
+
+    if st.session_state.processRank:
+        st.divider()
+        st.header('Output')
+        jobDescriptionRnk = uploadedJobDescriptionRnk.read().decode('utf-8')
+        resumeRnk = pd.read_excel(uploadedResumeRnk)
+        resumeRnk = rankResumes(jobDescriptionRnk, resumeRnk)
+        with st.expander('View Job Description'):
+            st.write(jobDescriptionRnk)
+        currentRnk = filterDataframeRnk(resumeRnk)
+        st.dataframe(currentRnk, use_container_width = True, hide_index = True)
+        xlsxRnk = convertDfToXlsx(currentRnk)
+        st.download_button(label='Save Current Output as XLSX', data = xlsxRnk, file_name = 'Resumes_ranked.xlsx')
+

knn_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69d9b5be2223b9acb49fe7cf16a6df47daa312ed35b21e2a9341b9ae97575c60
+size 4223478

label_encoder.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0ed5884fd4a68001090ffe82fda74566574d5b9aed8654eec48c19cc4585f1b
+size 715

nca_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fd5ab8d8adb40ead5421e8d90e36c99004f2af426be6659e7add2f0c58893e7
+size 43294492

requirements.txt

@@ -0,0 +1,67 @@
+altair==5.1.1
+attrs==23.1.0
+blinker==1.6.2
+cachetools==5.3.1
+certifi==2023.7.22
+charset-normalizer==3.2.0
+click==8.1.7
+contourpy==1.1.1
+cycler==0.11.0
+et-xmlfile==1.1.0
+fonttools==4.42.1
+gensim==4.3.2
+gitdb==4.0.10
+GitPython==3.1.37
+idna==3.4
+importlib-metadata==6.8.0
+Jinja2==3.1.2
+joblib==1.3.2
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+matplotlib==3.8.0
+mdurl==0.1.2
+nltk==3.8.1
+numpy==1.26.0
+openpyxl==3.1.2
+packaging==23.1
+pandas==2.1.1
+Pillow==9.5.0
+protobuf==4.24.3
+pyarrow==13.0.0
+pydeck==0.8.1b0
+Pygments==2.16.1
+pyparsing==3.1.1
+PyQt5==5.15.9
+PyQt5-Qt5==5.15.2
+PyQt5-sip==12.12.2
+python-dateutil==2.8.2
+pytz==2023.3.post1
+referencing==0.30.2
+regex==2023.8.8
+requests==2.31.0
+rich==13.5.3
+rpds-py==0.10.3
+scikit-learn==1.3.1
+scipy==1.11.2
+seaborn==0.12.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+streamlit==1.27.0
+tenacity==8.2.3
+threadpoolctl==3.2.0
+toml==0.10.2
+toolz==0.12.0
+tornado==6.3.3
+tqdm==4.66.1
+typing_extensions==4.8.0
+tzdata==2023.3
+tzlocal==5.0.1
+urllib3==2.0.5
+validators==0.22.0
+watchdog==3.0.0
+XlsxWriter==3.1.4
+zipp==3.17.0

tfidf_vectorizer.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:986d302aa10cddb969608ad2afe08fade436d19afd812ba508a0c8b4f1498a2b
+size 794455

train_classifier.py

@@ -0,0 +1,102 @@
+import joblib
+import matplotlib.pyplot as plt
+import pandas as pd
+import re
+import seaborn as sns
+from nltk.corpus import stopwords
+from nltk.stem import PorterStemmer
+from sklearn import metrics
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.neighbors import KNeighborsClassifier, NeighborhoodComponentsAnalysis
+from sklearn.preprocessing import LabelEncoder
+
+file_path = '~/Projects/hau/csstudy/resume-screening-and-classification/knn-trial/datasets/dataset_hr_edited.csv'
+
+resumeDataSet = pd.read_csv(file_path)
+
+stop_words = set(stopwords.words('english'))
+stemmer = PorterStemmer()
+
+print (resumeDataSet['Category'].value_counts())
+
+def cleanResume(resumeText):
+    resumeText = re.sub('http\S+\s*', ' ', resumeText)  # remove URLs
+    resumeText = re.sub('RT|cc', ' ', resumeText)  # remove RT and cc
+    resumeText = re.sub('#\S+', '', resumeText)  # remove hashtags
+    resumeText = re.sub('@\S+', '  ', resumeText)  # remove mentions
+    resumeText = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), ' ', resumeText)  # remove punctuations
+    resumeText = re.sub(r'[^\x00-\x7f]',r' ', resumeText) 
+    resumeText = re.sub('\s+', ' ', resumeText)  # remove extra whitespace
+
+    words = resumeText.split()
+    words = [word for word in words if word.lower() not in stop_words]
+    words = [stemmer.stem(word.lower()) for word in words if word.lower() not in stop_words]
+    resumeText = ' '.join(words)
+    return resumeText
+
+resumeDataSet['cleaned_resume'] = resumeDataSet.Resume.apply(lambda x: cleanResume(x))
+
+le = LabelEncoder()
+resumeDataSet['Category'] = le.fit_transform(resumeDataSet['Category'])
+le_filename = f'label_encoder.joblib'
+joblib.dump(le, le_filename)
+
+requiredText = resumeDataSet['cleaned_resume'].values
+requiredTarget = resumeDataSet['Category'].values
+
+word_vectorizer = TfidfVectorizer(
+    stop_words='english',
+    sublinear_tf=True,
+    max_features=18038
+)
+
+word_vectorizer.fit(requiredText)
+joblib.dump(word_vectorizer, 'tfidf_vectorizer.joblib')
+WordFeatures = word_vectorizer.transform(requiredText)
+
+nca = NeighborhoodComponentsAnalysis(n_components=300, random_state=42)
+WordFeatures = nca.fit_transform(WordFeatures.toarray(), requiredTarget)
+nca_filename = f'nca_model.joblib'
+joblib.dump(nca, nca_filename)
+
+X_train,X_test,y_train,y_test = train_test_split(WordFeatures,requiredTarget,random_state=42, test_size=0.2,shuffle=True, stratify=requiredTarget)
+print(X_train.shape)
+print(X_test.shape)
+
+# n_neighbors_values = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99]
+# weights = ["uniform", "distance"]
+# metric = ["euclidean", "manhattan", "minkowski", "cosine"]
+# algorithm = ['ball_tree', 'kd_tree', 'brute', 'auto']
+# param_grid = dict(n_neighbors=n_neighbors_values, weights=weights, metric=metric, algorithm=algorithm)
+# knn = KNeighborsClassifier()
+# gs = GridSearchCV(estimator=knn, param_grid=param_grid, scoring="accuracy", verbose=1, cv=10, n_jobs=3)
+# grid_search = gs.fit(X_train, y_train)
+# best_score = grid_search.best_score_
+# best_parameters = grid_search.best_params_
+# print("Best Score:", best_score)
+# print("Best Parameters:", best_parameters)
+
+knn = KNeighborsClassifier(n_neighbors=1, 
+                           metric='manhattan',
+                           weights='uniform',
+                           algorithm='ball_tree',
+                           )
+knn.fit(X_train, y_train)
+
+knnModel_filename = f'knn_model.joblib'
+joblib.dump(knn, knnModel_filename)
+
+prediction = knn.predict(X_test)
+print('Accuracy of KNeighbors Classifier on training set: {:.2f}'.format(knn.score(X_train, y_train)))
+print('Accuracy of KNeighbors Classifier on test set: {:.2f}'.format(knn.score(X_test, y_test)))
+print("\n Classification report for classifier %s:\n%s\n" % (knn, metrics.classification_report(y_test, prediction)))
+
+confusion_matrix = metrics.confusion_matrix(y_test, prediction)
+
+plt.figure(figsize=(10, 10))
+sns.heatmap(confusion_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=le.classes_, yticklabels=le.classes_)
+plt.xlabel('Predicted')
+plt.ylabel('True')
+plt.title('Confusion Matrix')
+plt.show()

utils.py

@@ -0,0 +1,602 @@
+import altair as alt
+# import datetime
+import joblib
+import nltk
+import numpy as np
+import pandas as pd
+import re
+import streamlit as st 
+import time
+
+from gensim.corpora import Dictionary
+from gensim.models import KeyedVectors, TfidfModel
+from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix, WordEmbeddingSimilarityIndex
+from io import BytesIO
+from nltk import pos_tag, word_tokenize
+from nltk.corpus import stopwords, wordnet
+from nltk.stem import PorterStemmer, WordNetLemmatizer
+from pandas.api.types import is_categorical_dtype, is_numeric_dtype
+from PIL import Image
+from scipy.sparse import csr_matrix, hstack
+
+nltk.download('averaged_perceptron_tagger')
+nltk.download('punkt')
+nltk.download('stopwords')
+
+stop_words = set(stopwords.words('english'))
+lemmatizer = WordNetLemmatizer()
+stemmer = PorterStemmer()
+
+def addZeroFeatures(matrix):
+    maxFeatures = 18038
+    numDocs, numTerms = matrix.shape
+    missingFeatures = maxFeatures - numTerms
+    if missingFeatures > 0:
+        zeroFeatures = csr_matrix((numDocs, missingFeatures), dtype=np.float64)
+        matrix = hstack([matrix, zeroFeatures])
+    return matrix
+
+@st.cache_data(max_entries = 1, show_spinner = False)
+def classifyResumes(df):
+    # WITH PROGRESS BAR
+    progressBar = st.progress(0)
+    progressBar.progress(0, text = "Preprocessing data ...")
+    startTime = time.time()
+    df['cleanedResume'] = df.Resume.apply(lambda x: performStemming(x))
+    resumeText = df['cleanedResume'].values
+    progressBar.progress(20, text = "Extracting features ...")
+    vectorizer = loadTfidfVectorizer()
+    wordFeatures = vectorizer.transform(resumeText)
+    wordFeaturesWithZeros = addZeroFeatures(wordFeatures)
+    progressBar.progress(40, text = "Reducing dimensionality ...")
+    finalFeatures = dimensionalityReduction(wordFeaturesWithZeros)
+    progressBar.progress(60, text = "Predicting categories ...")
+    knn = loadKnnModel()
+    predictedCategories = knn.predict(finalFeatures)
+    progressBar.progress(80, text = "Finishing touches ...")
+    le = loadLabelEncoder()
+    df['Industry Category'] = le.inverse_transform(predictedCategories)
+    df['Industry Category'] = pd.Categorical(df['Industry Category'])
+    df.drop(columns = ['cleanedResume'], inplace = True)
+    endTime = time.time()
+    elapsedSeconds = endTime - startTime
+    hours, remainder = divmod(int(elapsedSeconds), 3600)
+    minutes, _ = divmod(remainder, 60)
+    secondsWithDecimals = '{:.2f}'.format(elapsedSeconds % 60)
+    elapsedTimeStr = f'{hours} h : {minutes} m : {secondsWithDecimals} s'
+    progressBar.progress(100, text = f'Classification Complete!')
+    time.sleep(1)
+    progressBar.empty()
+    st.info(f'Finished classifying {len(resumeText)} resumes - {elapsedTimeStr}')
+    return df 
+
+    # NO LOADING WIDGET
+    # startTime = time.time()
+    # df['cleanedResume'] = df.Resume.apply(lambda x: performStemming(x))
+    # resumeText = df['cleanedResume'].values
+    # vectorizer = loadTfidfVectorizer()
+    # wordFeatures = vectorizer.transform(resumeText)
+    # wordFeaturesWithZeros = addZeroFeatures(wordFeatures)
+    # finalFeatures = dimensionalityReduction(wordFeaturesWithZeros)
+    # knn = loadKnnModel()
+    # predictedCategories = knn.predict(finalFeatures)
+    # le = loadLabelEncoder()
+    # df['Industry Category'] = le.inverse_transform(predictedCategories)
+    # df['Industry Category'] = pd.Categorical(df['Industry Category'])
+    # df.drop(columns = ['cleanedResume'], inplace = True)
+    # endTime = time.time()
+    # elapsedSeconds = endTime - startTime
+    # elapsedTime = datetime.timedelta(seconds = elapsedSeconds)
+    # hours, remainder = divmod(elapsedTime.seconds, 3600)
+    # minutes, seconds = divmod(remainder, 60)
+    # elapsedTimeStr = f"{hours} hr {minutes} min {seconds} sec"
+    # st.info(f'Finished in {elapsedTimeStr}')
+    # return df 
+
+def clickClassify():
+    st.session_state.processClf = True
+
+def clickRank():
+    st.session_state.processRank = True
+
+def convertDfToXlsx(df):
+    output = BytesIO()
+    writer = pd.ExcelWriter(output, engine = 'xlsxwriter')
+    df.to_excel(writer, index = False, sheet_name = 'Sheet1')
+    workbook = writer.book
+    worksheet = writer.sheets['Sheet1']
+    format1 = workbook.add_format({'num_format': '0.00'}) 
+    worksheet.set_column('A:A', None, format1)  
+    writer.close()
+    processedData = output.getvalue()
+    return processedData
+
+def createBarChart(df):
+    valueCounts = df['Industry Category'].value_counts().reset_index()
+    valueCounts.columns = ['Industry Category', 'Count']
+    newDataframe = pd.DataFrame(valueCounts)
+    barChart = alt.Chart(newDataframe,
+    ).mark_bar(
+        color = '#56B6C2',
+        size = 13 
+    ).encode(
+        x = alt.X('Count:Q', axis = alt.Axis(format = 'd'), title = 'Number of Resumes'),
+        y = alt.Y('Industry Category:N', title = 'Category'),
+        tooltip = ['Industry Category', 'Count']
+    ).properties(
+        title = 'Number of Resumes per Category',
+    )
+    return barChart
+
+def dimensionalityReduction(features):
+    nca = joblib.load('nca_model.joblib')
+    features = nca.transform(features.toarray())
+    return features    
+
+def filterDataframeClf(df: pd.DataFrame) -> pd.DataFrame:
+    modify = st.toggle("Add filters", key = 'filter-clf-1')
+    if not modify:
+        return df
+    df = df.copy()
+    modificationContainer = st.container()
+    with modificationContainer:
+        toFilterColumns = st.multiselect("Filter table on", df.columns, key = 'filter-clf-2')
+        for column in toFilterColumns:
+            left, right = st.columns((1, 20))
+            left.write("↳")
+            widgetKey = f'filter-clf-{toFilterColumns.index(column)}-{column}'
+            if is_categorical_dtype(df[column]):
+                userCatInput = right.multiselect(
+                    f'Values for {column}',
+                    df[column].unique(),
+                    default = list(df[column].unique()),
+                    key = widgetKey 
+                )
+                df = df[df[column].isin(userCatInput)]
+            elif is_numeric_dtype(df[column]):
+                _min = float(df[column].min())
+                _max = float(df[column].max())
+                step = (_max - _min) / 100
+                userNumInput = right.slider(
+                    f'Values for {column}',
+                    min_value = _min,
+                    max_value = _max,
+                    value = (_min, _max),
+                    step = step,
+                    key = widgetKey 
+                )
+                df = df[df[column].between(*userNumInput)]
+            else:
+                userTextInput = right.text_input(
+                    f'Substring or regex in {column}',
+                    key = widgetKey 
+                )
+                if userTextInput:
+                    userTextInput = userTextInput.lower()
+                    df = df[df[column].astype(str).str.lower().str.contains(userTextInput)]
+    return df
+
+def filterDataframeRnk(df: pd.DataFrame) -> pd.DataFrame:
+    modify = st.toggle("Add filters", key = 'filter-rnk-1')
+    if not modify:
+        return df
+    df = df.copy()
+    modificationContainer = st.container()
+    with modificationContainer:
+        toFilterColumns = st.multiselect("Filter table on", df.columns, key = 'filter-rnk-2')
+        for column in toFilterColumns:
+            left, right = st.columns((1, 20))
+            left.write("↳")
+            widgetKey = f'filter-rnk-{toFilterColumns.index(column)}-{column}'
+            if is_categorical_dtype(df[column]):
+                userCatInput = right.multiselect(
+                    f'Values for {column}',
+                    df[column].unique(),
+                    default = list(df[column].unique()),
+                    key = widgetKey
+                )
+                df = df[df[column].isin(userCatInput)]
+            elif is_numeric_dtype(df[column]):
+                _min = float(df[column].min())
+                _max = float(df[column].max())
+                step = (_max - _min) / 100
+                userNumInput = right.slider(
+                    f'Values for {column}',
+                    min_value = _min,
+                    max_value = _max,
+                    value = (_min, _max),
+                    step = step,
+                    key = widgetKey
+                )
+                df = df[df[column].between(*userNumInput)]
+            else:
+                userTextInput = right.text_input(
+                    f'Substring or regex in {column}',
+                    key = widgetKey
+                )
+                if userTextInput:
+                    userTextInput = userTextInput.lower()
+                    df = df[df[column].astype(str).str.lower().str.contains(userTextInput)]
+    return df
+
+def getWordnetPos(tag):
+    if tag.startswith('J'):
+        return wordnet.ADJ
+    elif tag.startswith('V'):
+        return wordnet.VERB
+    elif tag.startswith('N'):
+        return wordnet.NOUN
+    elif tag.startswith('R'):
+        return wordnet.ADV
+    else:
+        return wordnet.NOUN
+
+def loadKnnModel():
+    knnModelFileName = f'knn_model.joblib'
+    return joblib.load(knnModelFileName)
+
+def loadLabelEncoder():
+    labelEncoderFileName = f'label_encoder.joblib'
+    return joblib.load(labelEncoderFileName)
+
+def loadTfidfVectorizer():
+    tfidfVectorizerFileName = f'tfidf_vectorizer.joblib' 
+    return joblib.load(tfidfVectorizerFileName)
+
+def performLemmatization(text):
+    text = re.sub('http\S+\s*', ' ', text)
+    text = re.sub('RT|cc', ' ', text)
+    text = re.sub('#\S+', '', text)
+    text = re.sub('@\S+', '  ', text)
+    text = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), ' ', text)
+    text = re.sub(r'[^\x00-\x7f]',r' ', text)
+    text = re.sub('\s+', ' ', text)
+    words = word_tokenize(text)
+    words = [
+        lemmatizer.lemmatize(word.lower(), pos = getWordnetPos(pos)) 
+        for word, pos in pos_tag(words) if word.lower() not in stop_words
+    ]
+    return words
+
+def performStemming(text):
+    text = re.sub('http\S+\s*', ' ', text)
+    text = re.sub('RT|cc', ' ', text)
+    text = re.sub('#\S+', '', text)
+    text = re.sub('@\S+', '  ', text)
+    text = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), ' ', text)
+    text = re.sub(r'[^\x00-\x7f]',r' ', text)
+    text = re.sub('\s+', ' ', text)
+    words = word_tokenize(text)
+    words = [stemmer.stem(word.lower()) for word in words if word.lower() not in stop_words]
+    text = ' '.join(words)
+    return text 
+
+@st.cache_data
+def loadModel():
+    # model_path = '~/Projects/hau/csstudy/final-csstudy/wiki-news-300d-1M-subword.vec'
+    model_path = 'wiki-news-300d-1M-subword.vec'
+    model = KeyedVectors.load_word2vec_format(model_path, limit = 100000)
+    return model
+
+model = loadModel()
+
+@st.cache_data(max_entries = 1, show_spinner = False)
+def rankResumes(text, df):
+    # WITH PROGRESS BAR
+    progressBar = st.progress(0)
+    progressBar.progress(0, text = "Preprocessing data ...")
+    startTime = time.time()
+    jobDescriptionText = performLemmatization(text)
+    df['cleanedResume'] = df['Resume'].apply(lambda x: performLemmatization(x))
+    documents = [jobDescriptionText] + df['cleanedResume'].tolist()
+    progressBar.progress(13, text = "Creating a dictionary ...")
+    dictionary = Dictionary(documents)
+    progressBar.progress(25, text = "Creating a TF-IDF model ...")
+    tfidf = TfidfModel(dictionary = dictionary)
+    progressBar.progress(38, text = "Creating a Similarity Index...")
+    similarityIndex = WordEmbeddingSimilarityIndex(model)
+    progressBar.progress(50, text = "Creating a Similarity Matrix...")
+    similarityMatrix = SparseTermSimilarityMatrix(similarityIndex, dictionary, tfidf)
+    progressBar.progress(63, text = "Setting up job description as the query ...")
+    query = tfidf[dictionary.doc2bow(jobDescriptionText)]
+    progressBar.progress(75, text = "Calculating semantic similarities ...")
+    index = SoftCosineSimilarity(
+        tfidf[[dictionary.doc2bow(resume) for resume in df['cleanedResume']]],
+        similarityMatrix 
+    )
+    similarities = index[query]
+    progressBar.progress(88, text = "Finishing touches ...")
+    df['Similarity Score'] = similarities
+    df['Rank'] = df['Similarity Score'].rank(ascending=False, method='dense').astype(int)
+    df.sort_values(by='Rank', inplace=True)
+    df.drop(columns = ['cleanedResume'], inplace = True)
+    endTime = time.time()
+    elapsedSeconds = endTime - startTime
+    hours, remainder = divmod(int(elapsedSeconds), 3600)
+    minutes, _ = divmod(remainder, 60)
+    secondsWithDecimals = '{:.2f}'.format(elapsedSeconds % 60)
+    elapsedTimeStr = f'{hours} h : {minutes} m : {secondsWithDecimals} s'
+    progressBar.progress(100, text = f'Classification Complete!')
+    time.sleep(1)
+    progressBar.empty()
+    st.info(f'Finished ranking {len(df)} resumes - {elapsedTimeStr}')
+    return df 
+    
+    # NO LOADING WIDGET
+    # startTime = time.time()
+    # jobDescriptionText = performLemmatization(text)
+    # df['cleanedResume'] = df['Resume'].apply(lambda x: performLemmatization(x))
+    # documents = [jobDescriptionText] + df['cleanedResume'].tolist()
+    # dictionary = Dictionary(documents)
+    # tfidf = TfidfModel(dictionary = dictionary)
+    # similarityIndex = WordEmbeddingSimilarityIndex(model)
+    # similarityMatrix = SparseTermSimilarityMatrix(similarityIndex, dictionary, tfidf)
+    # query = tfidf[dictionary.doc2bow(jobDescriptionText)]
+    # index = SoftCosineSimilarity(
+    #     tfidf[[dictionary.doc2bow(resume) for resume in df['cleanedResume']]],
+    #     similarityMatrix 
+    # )
+    # similarities = index[query]
+    # df['Similarity Score'] = similarities
+    # df.sort_values(by = 'Similarity Score', ascending = False, inplace = True)
+    # df.drop(columns = ['cleanedResume'], inplace = True)
+    # endTime = time.time()
+    # elapsedSeconds = endTime - startTime
+    # elapsedTime = datetime.timedelta(seconds = elapsedSeconds)
+    # hours, remainder = divmod(elapsedTime.seconds, 3600)
+    # minutes, seconds = divmod(remainder, 60)
+    # elapsedTimeStr = f"{hours} hr {minutes} min {seconds} sec"
+    # st.info(f'Finished in {elapsedTimeStr}')
+    # return df 
+
+# TF-IDF + LSA + COSSIM
+# from sklearn.decomposition import TruncatedSVD
+# import math
+# def resumesRank(jobDescriptionRnk, resumeRnk):
+#     jobDescriptionRnk = preprocessing(jobDescriptionRnk)
+#     resumeRnk['cleanedResume'] = resumeRnk.Resume.apply(lambda x: preprocessing(x))
+#     resumes = resumeRnk['cleanedResume'].values
+#     # tfidfVectorizer = TfidfVectorizer(sublinear_tf = True, stop_words = 'english')
+#     # tfidfVectorizer = TfidfVectorizer(sublinear_tf = True)
+#     # tfidfVectorizer = TfidfVectorizer(stop_words = 'english')
+#     tfidfVectorizer = TfidfVectorizer()
+#     tfidfMatrix = tfidfVectorizer.fit_transform([jobDescriptionRnk] + list(resumes))
+#     num_features = len(tfidfVectorizer.get_feature_names_out())
+#     st.write(f"Number of TF-IDF Features: {num_features}")
+#     nComponents = math.ceil(len(resumes) * 0.55)
+#     # nComponents = math.ceil(num_features * 0.01)
+#     # nComponents = 5 
+#     st.write(nComponents)
+#     # nComponents = len(resumes)
+#     lsa = TruncatedSVD(n_components=nComponents)
+#     lsaMatrix = lsa.fit_transform(tfidfMatrix)
+#     similarityScores = cosine_similarity(lsaMatrix[0:1], lsaMatrix[1:])
+#     resumeRnk['Similarity Score (%)'] = similarityScores[0] * 100
+#     resumeRnk = resumeRnk.sort_values(by='Similarity Score (%)', ascending=False)
+#     del resumeRnk['cleanedResume']
+#     return resumeRnk
+
+# 1 BY 1 SOFT COSSIM
+# def resumesRank(jobDescriptionRnk, resumeRnk):
+#     jobDescriptionText = preprocessing2(jobDescriptionRnk)
+#     resumeRnk['cleanedResume'] = resumeRnk['Resume'].apply(lambda x: preprocessing2(x))
+#     similarityscore = []
+#     for resume in resumeRnk['cleanedResume']:
+#         documents = [jobDescriptionText, resume] 
+#         dictionary = Dictionary(documents)
+#         documentBow = [dictionary.doc2bow(doc) for doc in documents]
+#         tfidf = TfidfModel(documentBow, dictionary=dictionary)
+#         similarityIndex = WordEmbeddingSimilarityIndex(model)
+#         similarityMatrix = SparseTermSimilarityMatrix(similarityIndex, dictionary, tfidf)
+#         # similarityMatrix = SparseTermSimilarityMatrix(similarityIndex, dictionary)
+#         value = tfidf[dictionary.doc2bow(resume)]
+#         # value = dictionary.doc2bow(jobDescriptionText)
+#         index = SoftCosineSimilarity(
+#             # tfidf[[dictionary.doc2bow(resume)]], 
+#             tfidf[[dictionary.doc2bow(jobDescriptionText)]], 
+#             # [dictionary.doc2bow(resume) for resume in resumeRnk['cleanedResume']],
+#             similarityMatrix, 
+#         )
+#         similarities = index[value]
+#         similarityscore.append(similarities)
+#     print(similarityscore)
+#     resumeRnk['Similarity Score'] = similarityscore 
+#     resumeRnk.sort_values(by='Similarity Score', ascending=False, inplace=True)
+#     resumeRnk.drop(columns=['cleanedResume'], inplace=True)
+#     return resumeRnk
+#
+# TF-IDF SCORE + WORD EMBEDDINGS SCORE
+# def resumesRank(jobDescriptionRnk, resumeRnk):
+#     def get_word_embedding(text):
+#         words = text.split()
+#         valid_words = [word for word in text.split() if word in model]
+#         if valid_words:
+#             return np.mean([model[word] for word in valid_words], axis=0)
+#         else:
+#             return np.zeros(model.vector_size)
+#     jobDescriptionRnk = preprocessing2(jobDescriptionRnk)
+#     resumeRnk['cleanedResume'] = resumeRnk.Resume.apply(lambda x: preprocessing2(x))
+#     tfidfVectorizer = TfidfVectorizer(sublinear_tf = True, stop_words='english')
+#     jobTfidf = tfidfVectorizer.fit_transform([jobDescriptionRnk])
+#     jobDescriptionEmbedding = get_word_embedding(jobDescriptionRnk)
+#     resumeSimilarities = []
+#     for resumeContent in resumeRnk['cleanedResume']:
+#         resumeEmbedding = get_word_embedding(resumeContent)
+#         similarityFastText = cosine_similarity([jobDescriptionEmbedding], [resumeEmbedding])[0][0]
+#         similarityTFIDF = cosine_similarity(jobTfidf, tfidfVectorizer.transform([resumeContent]))[0][0]
+#         similarity = (0.6 * similarityTFIDF) + (0.4 * similarityFastText)
+#         final_similarity = similarity * 100
+#         resumeSimilarities.append(final_similarity)
+#     resumeRnk['Similarity Score (%)'] = resumeSimilarities
+#     resumeRnk = resumeRnk.sort_values(by='Similarity Score (%)', ascending=False)
+#     del resumeRnk['cleanedResume']
+#     return resumeRnk
+
+# WORD EMBEDDINGS + COSSIM
+# def resumesRank(jobDescriptionRnk, resumeRnk):
+#     def get_word_embedding(text):
+#         words = text.split()
+#         valid_words = [word for word in text.split() if word in model]
+#         if valid_words:
+#             return np.mean([model[word] for word in valid_words], axis=0)
+#         else:
+#             return np.zeros(model.vector_size)
+#     jobDescriptionRnk = preprocessing2(jobDescriptionRnk)
+#     jobDescriptionEmbedding = get_word_embedding(jobDescriptionRnk)
+#     resumeRnk['cleanedResume'] = resumeRnk.Resume.apply(lambda x: preprocessing2(x))
+#     resumeSimilarities = []
+#     for resumeContent in resumeRnk['cleanedResume']:
+#         resumeEmbedding = get_word_embedding(resumeContent)
+#         similarity = cosine_similarity([jobDescriptionEmbedding], [resumeEmbedding])[0][0]
+#         percentageSimilarity = similarity * 100
+#         resumeSimilarities.append(percentageSimilarity)
+#     resumeRnk['Similarity Score (%)'] = resumeSimilarities
+#     resumeRnk = resumeRnk.sort_values(by='Similarity Score (%)', ascending=False)
+#     del resumeRnk['cleanedResume']
+#     return resumeRnk
+
+# TF-IDF + COSSIM
+# def resumesRank(jobDescriptionRnk, resumeRnk):
+#     jobDescriptionRnk = preprocessing2(jobDescriptionRnk)
+#     resumeRnk['cleanedResume'] = resumeRnk.Resume.apply(lambda x: preprocessing2(x))
+#     tfidfVectorizer = TfidfVectorizer(sublinear_tf = True, stop_words='english')
+#     jobTfidf = tfidfVectorizer.fit_transform([jobDescriptionRnk])
+#     resumeSimilarities = []
+#     for resumeContent in resumeRnk['cleanedResume']:
+#         resumeTfidf = tfidfVectorizer.transform([resumeContent])
+#         similarity = cosine_similarity(jobTfidf, resumeTfidf)
+#         percentageSimilarity = (similarity[0][0] * 100)
+#         resumeSimilarities.append(percentageSimilarity)
+#     resumeRnk['Similarity Score (%)'] = resumeSimilarities
+#     resumeRnk = resumeRnk.sort_values(by='Similarity Score (%)', ascending=False)
+#     del resumeRnk['cleanedResume']
+#     return resumeRnk
+
+def writeGettingStarted():
+    st.write("""
+    ## Hello, Welcome!  
+    In today's competitive job market, the process of manually screening resumes has become a daunting task for recruiters and hiring managers. 
+    The sheer volume of applications received for a single job posting can make it extremely time-consuming to identify the most suitable candidates efficiently. 
+    This often leads to missed opportunities and the potential loss of top-tier talent.
+
+    The ***Resume Screening & Classification*** website application aims to help alleviate the challenges posed by manual resume screening. 
+    The main objectives are:
+    - To classify the resumes into their most suitable job industry category
+    - To compare the resumes to the job description and rank them by similarity
+    """)
+    st.divider()
+    st.write("""
+    ## Input Guide 
+    #### For the Job Description: 
+    Ensure the job description is saved in a text (.txt) file. 
+    Kindly outline the responsibilities, qualifications, and skills associated with the position.
+
+    #### For the Resumes: 
+    Resumes must be compiled in an excel (.xlsx) file. 
+    The organization of columns is up to you but ensure that the "Resume" column is present.
+    The values under this column should include all the relevant details for each resume.
+    """)
+    st.divider()
+    st.write("""
+    ## Demo Walkthrough
+    #### Classify Tab:
+    The web app will classify the resumes into their most suitable job industry category.
+    Currently the Category Scope consists of the following:
+    """)
+    column1, column2 = st.columns(2)
+    with column1:
+        st.write("""
+        - Aviation
+        - Business development
+        - Culinary
+        - Education
+        - Engineering
+        - Finance
+        """)
+    with column2:
+        st.write("""
+        - Fitness
+        - Healthcare
+        - HR
+        - Information Technology
+        - Public relations
+        """)
+    with st.expander('Classification Steps'):
+        st.write("""
+        ##### Upload Resumes & Start Processing:
+        - Navigate to the "Classify" tab.
+        - Click the "Upload Resumes" button.
+        - Select the Excel file (.xlsx) containing the resumes you want to classify. Ensure that your Excel file has the "Resume" column with the resume text and any necessary columns for filtering or additional information.
+        - Click the "Start Processing" button.
+        - The app will analyze the resumes and categorize them into job industry categories.
+        ######
+        """)
+        imgClf1 = Image.open('clf-1.png')
+        st.image(imgClf1, use_column_width = True, output_format = "PNG")
+        st.write("""
+        ##### View Bar Chart:
+        - A bar chart will appear, showing the number of resumes per category, helping you visualize the distribution.
+        ######
+        """)
+        imgClf2 = Image.open('clf-2.png')
+        st.image(imgClf2, use_column_width = True, output_format = "PNG")
+        st.write("""
+        ##### Add Filters:
+        - You can apply filters to the dataframe to narrow down your results.
+        ######
+        """)
+        imgClf3 = Image.open('clf-3.png')
+        st.image(imgClf3, use_column_width = True, output_format = "PNG")
+        st.write("""
+        ##### Donwload Results:
+        - Once you've applied filters or are satisfied with the results, you can download the current dataframe as an Excel file by clicking the "Save Current Output as XLSX" button.
+        ####
+        """)
+        imgClf4 = Image.open('clf-4.png')
+        st.image(imgClf4, use_column_width = True, output_format = "PNG")
+    st.write("""
+    #### Rank Tab:
+    The web app will rank the resumes based on their semantic similarity to the job description. 
+    The similarity score ranges from -1 to 1.
+    A score of 1 is achieved when Document A and Document B are identical.
+
+    ##### **Kindly take note:**
+
+    It's important to note that these scores are not absolute and may change when more resumes are added in the comparison.
+    The ranking algorithm dynamically adjusts its results based on the entire set of uploaded resumes.
+    We recommend considering the scores as a relative measure rather than an absolute determination.
+    """)
+    with st.expander('Ranking Steps'):
+        st.write("""
+        ##### Upload Files & Start Processing:
+        - Navigate to the "Rank" tab.
+        - Upload the job description as a text file. This file should contain the description of the job you want to compare resumes against.
+        - Upload the Excel file that contains the resumes you want to rank.
+        - Click the "Start Processing" button.
+        - The app will analyze the job description and rank the resumes based on their similarity to the job description.
+        ######
+        """)
+        imgRnk1 = Image.open('rnk-1.png')
+        st.image(imgRnk1, use_column_width = True, output_format = "PNG")
+        st.write("""
+        ##### View Job Description:
+        - The output will display the contents of the job description for reference.
+        ######
+        """)
+        imgRnk2 = Image.open('rnk-2.png')
+        st.image(imgRnk2, use_column_width = True, output_format = "PNG")
+        st.write("""
+        ##### Add Filters:
+        - You can apply filters to the dataframe to narrow down your results.
+        ######
+        """)
+        imgRnk3 = Image.open('rnk-3.png')
+        st.image(imgRnk3, use_column_width = True, output_format = "PNG")
+        st.write("""
+        ##### Donwload Results:
+        - Once you've applied filters or are satisfied with the results, you can download the current dataframe as an Excel file by clicking the "Save Current Output as XLSX" button.
+        ####
+        """)
+        imgRnk4 = Image.open('rnk-4.png')
+        st.image(imgRnk4, use_column_width = True, output_format = "PNG")
+

wiki-news-300d-1M-subword.vec

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47e336cc848c697486273a9cb761ad30a3bf741d4ab26e779435bfa51491730e
+size 2256858740