Upload 17 files (#7)

- Upload 17 files (fd7f9d5220cde18cd4fd865209b63f2b5d9b1c37)

app.py

@@ -59,6 +59,12 @@ decoder_models_choice = ["GPT-3.5 Turbo", "Vicuna-7B"]
 with st.sidebar:
     st.subheader("Select Options:")
 
+    use_bm25 = st.checkbox("Use 2-Stage Retrieval (BM25)", value=True)
+
+    use_keyword_matching = st.checkbox(
+        "Use Exact Keyword Matching", value=False
+    )
+
     num_results = int(
         st.number_input("Number of Results to query", 1, 15, value=4)
     )
@@ -74,7 +80,6 @@ with st.sidebar:
         )
     )
 
-    use_bm25 = st.checkbox("Use 2-Stage Retrieval (BM25)", value=True)
     num_candidates = int(
         st.number_input(
             "Number of Candidates to Generate:",
@@ -84,9 +89,6 @@ with st.sidebar:
             value=50,
         )
     )
-    decoder_model = st.selectbox(
-        "Select Text Generation Model", decoder_models_choice
-    )
 
 
 col1, col2 = st.columns([3, 3], gap="medium")
@@ -94,9 +96,10 @@ col1, col2 = st.columns([3, 3], gap="medium")
 with col1:
     query_text = st.text_area(
         "Input Query",
-        value="How has the growth been for AMD in the PC market in 2020?",
+        value="How has the growth been for AMD in the PC market in Q1 and Q2 2020?",
     )
 
+
 # Extracting Document Entities from Question
 (
     companies,
@@ -116,11 +119,28 @@ ticker_year_quarter_tuples_list = ticker_year_quarter_tuples_creator(
     ticker_list, year_quarter_range_list
 )
 
+with col2:
+    if ticker_year_quarter_tuples_list != []:
+        st.markdown("**Companies mentioned in the question:**")
+        for i in ticker_list:
+            st.markdown("- " + i)
+        st.write("**Duration:**")
+        st.write(f"{start_quarter} {start_year} - {end_quarter} {end_year}")
+
 
 # Extract keywords from query
 all_keywords = extract_entities_keywords(query_text, vicuna_ner_2_model)
 if all_keywords != []:
     keywords = clean_keywords_all_combs(all_keywords)
+    store_keywords = keywords.copy()
+else:
+    keywords = None
+
+# Setting Keywords to None if use_keywords is False
+
+
+if use_keyword_matching == True:
+    keywords = store_keywords
 else:
     keywords = None
 
@@ -135,9 +155,7 @@ pinecone.init(
 pinecone_index_name = "week13-instructor-xl"
 pinecone_index = pinecone.Index(pinecone_index_name)
 retriever_model = get_instructor_embedding_model_api()
-instruction = (
-    "Represent the financial question for retrieving supporting documents:"
-)
+instruction = "Represent the finance query for retrieving related documents:"
 
 
 dense_query_embedding = create_dense_embeddings(
@@ -148,8 +166,9 @@ context_group = []
 if ticker_year_quarter_tuples_list != []:
     for ticker, quarter, year in ticker_year_quarter_tuples_list:
         if use_bm25 == True:
+            # Setting Ticker, Quarter, Year=None to trigger global bm25
             indices = get_indices_bm25(
-                data, ticker, quarter, year, num_candidates
+                data, query_text, None, None, None, num_candidates
             )
         else:
             indices = None
@@ -194,6 +213,12 @@ with col1:
         label="Model Prompt", value=prompt, height=400
     )
 
+with st.sidebar:
+    decoder_model = st.selectbox(
+        "Select Text Generation Model", decoder_models_choice
+    )
+
+
 if decoder_model == "GPT-3.5 Turbo":
     with col2:
         with st.form("gpt_form"):
@@ -224,9 +249,12 @@ if decoder_model == "GPT-3.5 Turbo":
 
 if decoder_model == "Vicuna-7B":
     with col2:
-        st.write("The Vicuna Model is running: ...")
-        st.write("The model takes 10-15 mins to generate the text.")
-        generated_text = vicuna_text_generate(prompt, vicuna_text_gen_model)
+        with st.spinner(
+            text="The Vicuna Model is running. The model takes approximately 10-15 mins to generate the text."
+        ):
+            generated_text = vicuna_text_generate(
+                prompt, vicuna_text_gen_model
+            )
         st.subheader("Answer:")
         regex_pattern_sentences = "(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s"
         generated_text_list = re.split(regex_pattern_sentences, generated_text)

utils/entity_extraction.py

@@ -35,8 +35,9 @@ def extract_entities_docs(query, model):
     """
     prompt = generate_ner_docs_prompt(query)
     string_of_dict = model.predict(prompt, api_name="/predict")
-
-    entities_dict = literal_eval(string_of_dict)
+    print(string_of_dict)
+    string_of_dict = string_of_dict.strip()
+    entities_dict = literal_eval(f"""{string_of_dict}""")
     start_quarter, start_year = entities_dict["start-duration"]
     end_quarter, end_year = entities_dict["end-duration"]
     companies = entities_dict["companies"]
@@ -176,8 +177,9 @@ def extract_entities_keywords(query, model):
     """
     prompt = generate_ner_keywords_prompt(query)
     string_of_dict = model.predict(prompt, api_name="/predict")
-
-    entities_dict = literal_eval(string_of_dict)
+    print(string_of_dict)
+    string_of_dict = string_of_dict.strip()
+    entities_dict = literal_eval(f"""{string_of_dict}""")
     keywords_list = entities_dict["entities"]
     return keywords_list
 

utils/retriever.py

@@ -55,6 +55,7 @@ def query_pinecone(
     filter_dict = {
         "QA_Flag": {"$eq": "Answer"},
     }
+    
     if year is not None:
         filter_dict["Year"] = int(year)
     if quarter is not None:
@@ -66,6 +67,7 @@ def query_pinecone(
     if indices is not None:
         filter_dict["index"] = {"$in": indices}
 
+    print(filter_dict)
     xc = index.query(
         vector=dense_vec,
         top_k=top_k,

utils/transcript_retrieval.py

@@ -2,30 +2,19 @@
 
 
 def retrieve_transcript(data, year, quarter, ticker):
-    if year == "All" or quarter == "All":
-        row = (
-            data.loc[
-                (data.Ticker == ticker),
-                ["File_Name"],
-            ]
-            .drop_duplicates()
-            .iloc[0, 0]
-        )
-    else:
-        row = (
-            data.loc[
-                (data.Year == int(year))
-                & (data.Quarter == quarter)
-                & (data.Ticker == ticker),
-                ["File_Name"],
-            ]
-            .drop_duplicates()
-            .iloc[0, 0]
-        )
+    print(year, quarter, ticker)
+    row = data.loc[
+        (data.Year == int(year))
+        & (data.Quarter == quarter)
+        & (data.Ticker == ticker),
+        ["File_Name"],
+    ]
+    filename = row.iloc[0, 0]
+    print(filename)
     # convert row to a string and join values with "-"
     # row_str = "-".join(row.astype(str)) + ".txt"
     open_file = open(
-        f"Transcripts/{ticker}/{row}",
+        f"Transcripts/{ticker}/{filename}",
         "r",
     )
     file_text = open_file.read()