current updates

app.py

@@ -5,6 +5,8 @@ import pandas as pd
 from collections import defaultdict
 import json
 import copy
+import re
+import tqdm
 import plotly.express as px
 from find_splitting_words import find_dividing_words
 
@@ -49,7 +51,39 @@ def get_current_data():
     # return the data as a CSV pandas
     return convert_df(pd.DataFrame(cur_query_data))
 
+@st.cache_data
+def escape_markdown(text):
+    # List of characters to escape
+    # Adding backslash to the list of special characters to escape itself as well
+    text = text.replace("``", "\"")
+    special_chars = ['\\', '`', '*', '_', '{', '}', '[', ']', '(', ')', '#', '+', '-', '.', '!', '|', "$"]
+    
+    # Escaping each special character
+    escaped_text = "".join(f"\\{char}" if char in special_chars else char for char in text)
+    
+    return escaped_text
 
+@st.cache_data
+def highlight_text(text, splitting_words):
+    # remove anything that will mess up markdown
+    text = escape_markdown(text)
+    changed = False
+    if not len(splitting_words):
+        return text, changed
+    
+    def replace_function(match):
+        return f'<span style="background-color: #FFFF00">{match.group(0)}</span>'
+    
+    # Compile a single regular expression pattern for all splitting words
+    pattern = '|'.join([re.escape(word) for word in splitting_words])
+    
+    # Perform case-insensitive replacement
+    new_text, num_subs = re.subn(pattern, replace_function, text, flags=re.IGNORECASE)
+    
+    if num_subs > 0:
+        changed = True
+
+    return new_text, changed
 
 
 if 'cur_instance_num' not in st.session_state:
@@ -94,7 +128,7 @@ with st.sidebar:
 
     z = st.header("Analysis Options")
     # sliderbar of how many Top N to choose
-    n_relevant_docs = st.slider("Number of relevant docs", 1, 999, 20)
+    n_relevant_docs = st.slider("Number of relevant docs", 1, 999, 300)
 
 
 col1, col2 = st.columns([1, 3], gap="large")
@@ -169,15 +203,39 @@ if corpus is not None and queries is not None and qrels is not None:
             # relevant
             relevant_docs = list(qrels[str(inst_num)].keys())[:n_relevant_docs]
             doc_texts = [(doc_id, corpus[doc_id]["title"] if "title" in corpus[doc_id] else "", corpus[doc_id]["text"]) for doc_id in relevant_docs]
-            splitting_words = find_dividing_words(relevant_docs_text)
+            splitting_words = find_dividing_words([item[1] + " " + item[2] for item in doc_texts])
 
-            st.markdown(splitting_words)
+            # make a selectbox of these splitting words (allow multiple)
+            container.subheader("Splitting Words")
+            container.text("Select words that are relevant to the query")
+            splitting_word_select = container.multiselect("Splitting Words", splitting_words, key="splitting_words")
+            container.divider()
 
-            container.subheader(f"Relevant Documents ({len(list(qrels[str(inst_num)].keys()))})")
             current_checkboxes = []
-            for (docid, title, text) in doc_texts:
+            total_changed = 0
+            highlighted_texts = []
+            highlighted_titles = []
+            for (docid, title, text) in tqdm.tqdm(doc_texts):
+                if not len(splitting_word_select):
+                    highlighted_texts.append(text)
+                    highlighted_titles.append(title)
+                    continue
+                highlighted_text, changed_text = highlight_text(text, splitting_word_select)
+                highlighted_title, changed_title = highlight_text(title, splitting_word_select)
+                highlighted_titles.append(highlighted_title)
+                highlighted_texts.append(highlighted_text)
+                total_changed += int(int(changed_text) or int(changed_title))
+
+            container.subheader(f"Relevant Documents ({len(list(qrels[str(inst_num)].keys()))})")
+            container.subheader(f"Total have these words: {total_changed}")
+
+            container.divider()
+                
+            for i, (docid, title, text) in enumerate(doc_texts):
+                container.markdown(f"## {docid}")
+                container.markdown(f"#### {highlighted_titles[i]}", True)
+                container.markdown(f"\n{highlighted_texts[i]}", True)
                 current_checkboxes.append((docid, container.checkbox(f'{docid} is Non-Relevant', key=docid)))
-                container.text_area(f"{docid}:", text)
 
 
             container.divider()

find_splitting_words.py

@@ -6,11 +6,14 @@ from nltk.tokenize import word_tokenize
 from collections import Counter
 import string
 import os
+import streamlit as st
 
 # Ensure you've downloaded the set of stop words the first time you run this
 import nltk
-nltk.download('punkt')
-nltk.download('stopwords')
+# only download if they don't exist
+if not os.path.exists(os.path.join(nltk.data.find('corpora'), 'stopwords')):
+    nltk.download('punkt')
+    nltk.download('stopwords')
 
 def preprocess_document(doc):
     """
@@ -30,6 +33,7 @@ def preprocess_document(doc):
     stemmed_tokens = [stemmer.stem(word) for word in filtered_tokens]
     return stemmed_tokens
 
+@st.cache_data
 def find_dividing_words(documents):
     """
     Identifies candidate words that might split the set of documents into two groups.
@@ -37,10 +41,13 @@ def find_dividing_words(documents):
     all_words = []
     per_doc_word_counts = []
     
+    i = 0
     for doc in documents:
+        print(i)
         preprocessed_doc = preprocess_document(doc)
         all_words.extend(preprocessed_doc)
         per_doc_word_counts.append(Counter(preprocessed_doc))
+        i += 1
     
     # Overall word frequency
     overall_word_counts = Counter(all_words)
@@ -50,8 +57,9 @@ def find_dividing_words(documents):
     candidate_words = []
     for word, count in overall_word_counts.items():
         doc_frequency = sum(1 for doc_count in per_doc_word_counts if doc_count[word] > 0)
-        if 0.3 * num_docs <= doc_frequency <= 0.7 * num_docs:
+        if 0.35 * num_docs <= doc_frequency <= 0.75 * num_docs:
             candidate_words.append(word)
+    print("Done with dividing words")
     
     return candidate_words
 

requirements.txt

@@ -3,4 +3,5 @@ streamlit==1.24.1
 plotly==5.15.0
 protobuf==3.20.0
 beautifulsoup4==4.12.2
-nltk==3.7
+nltk==3.7
+tqdm