POC for passive monitoring

app.py

@@ -9,15 +9,15 @@ from whoosh.query import Or, Term
 
 import csv
 
-THREAD_ID = "thread_id"
-
-logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=os.environ.get("LOGLEVEL", "INFO"))
 import gradio as gr
 from llama_index import ServiceContext, \
     Document, GPTListIndex, VectorStoreIndex
 from whoosh import fields, index
 from whoosh.qparser import QueryParser
 
+logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=os.environ.get("LOGLEVEL", "INFO"))
+THREAD_ID = "thread_id"
+
 thread_index = {}
 comment_index = {}
 llama_cache = {}
@@ -99,7 +99,7 @@ def create_comment_index(data_set):
         writer = write_ix.writer()
 
         # Read the CSV file and add documents to the index
-        count= 0
+        count = 0
         with open(f'csv/{data_set}.csv', 'r') as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
@@ -115,10 +115,10 @@ def create_comment_index(data_set):
     comment_index[data_set] = read_ix
 
 
-def search_keyword_matches(ix, input):
+def search_keyword_matches(ix, keyword_query):
     # Create a query parser
     query_parser = QueryParser("content", ix.schema)
-    query = query_parser.parse(input)
+    query = query_parser.parse(keyword_query)
     return execute_text_search(ix, query)
 
 
@@ -227,6 +227,7 @@ def remove_leading_numbers(text):
     # Use re.sub to replace any pattern of "<number>." at the beginning of a line.
     return re.sub(r'^\d+[.)]\s*', '', text, flags=re.M)
 
+
 def find_topics_with_llama(index_name, query, matches):
     print(f"Found {len(matches)} matches for {query}")
     docs_list = []

requirements.txt

@@ -10,4 +10,7 @@ slack~=0.0.2
 fastapi~=0.100.0
 pydantic>=1.10.5,<2
 
-ratelimit~=2.2.1
+ratelimit~=2.2.1
+Whoosh~=2.7.4
+spacy~=3.6.1
+numpy~=1.25.1