from typing import List, Tuple import pandas as pd from sentence_transformers import SentenceTransformer, util import streamlit as st from st_aggrid import AgGrid, GridOptionsBuilder, JsCode st.set_page_config(layout='wide') @st.cache(allow_output_mutation=True) def load_model(): return SentenceTransformer('all-MiniLM-L6-v2') def semantic_search(model, sentence, corpus_embeddings): query_embeddings = model.encode(sentence, convert_to_tensor=True, normalize_embeddings=True) hits = util.semantic_search(query_embeddings, corpus_embeddings, top_k=len(corpus_embeddings), score_function=util.dot_score) return pd.DataFrame(hits[0]) def top_k_similarity(model, df, query, corpus_embeddings): hits = semantic_search(model, [query], corpus_embeddings) result = pd.merge(df, hits, left_on='ID', right_on='corpus_id') result.sort_values(by='score', ascending=False, inplace=True) return result @st.cache(allow_output_mutation=True) def create_embedding(model: SentenceTransformer, data: pd.DataFrame, key: str) -> Tuple[list, list]: """Create vector embeddings from the dataset""" corpus_sentences = data[key].astype(str).tolist() corpus_embeddings = model.encode(sentences=corpus_sentences, show_progress_bar=True, convert_to_tensor=True, normalize_embeddings=True) return corpus_embeddings def load_dataset(columns: List) -> pd.DataFrame: """Load real-time dataset from google sheets""" sheet_id = '1KeuPPVw9gueNmMrQXk1uGFlY9H1vvhErMLiX_ZVRv_Y' sheet_name = 'Form Response 3'.replace(' ', '%20') url = f'https://docs.google.com/spreadsheets/d/{sheet_id}/gviz/tq?tqx=out:csv&sheet={sheet_name}' data = pd.read_csv(url) data = data.iloc[: , :7] data.columns = columns data.insert(0, 'ID', range(len(data))) return data def show_aggrid_table(result: pd.DataFrame): gb = GridOptionsBuilder.from_dataframe(result) gb.configure_pagination(paginationAutoPageSize=True) gb.configure_side_bar() gb.configure_default_column(min_column_width=200) gb.configure_selection('multiple', use_checkbox=True, groupSelectsChildren="Group checkbox select children") gb.configure_column(field="LinkedIn Profile", headerName="LinkedIn Profile", cellRenderer=JsCode('''function(params) {return `${params.value}`}''')) gridOptions = gb.build() grid_response = AgGrid( dataframe=result, gridOptions=gridOptions, height=1100, fit_columns_on_grid_load=True, data_return_mode='AS_INPUT', update_mode='VALUE_CHANGED', theme='light', enable_enterprise_modules=True, allow_unsafe_jscode=True, ) def main(): st.title('Job Posting Similarity') st.write('This app will help you find similar job titles real-time from ecommurz google sheets.') columns = ['Timestamp', 'Full Name', 'Company', 'Previous Role', 'Experience', 'Last Day', 'LinkedIn Profile'] data = load_dataset(columns) model = load_model() corpus_embeddings = create_embedding(model, data, 'Previous Role') job_title = st.text_input('Insert the job title below:', '') submitted = st.button('Submit') if submitted: st.info(f'Showing results for {job_title}') result = top_k_similarity(model, data, job_title, corpus_embeddings) result = result[columns] st.download_button( "Download Table", result.to_csv().encode('utf-8'), "result.csv", "text/csv", key='download-csv' ) show_aggrid_table(result) if __name__ == '__main__': main()